@crawlee/utils 4.0.0-beta.105 → 4.0.0-beta.107

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/internals/robots.d.ts +1 -3
  2. package/internals/robots.js +9 -9
  3. package/internals/sitemap.js +42 -42
  4. package/package.json +4 -4
package/internals/robots.d.ts CHANGED
@@ -19,9 +19,7 @@ import { Sitemap } from './sitemap.js';
19
19
  * ```
20
20
  */
21
21
  export declare class RobotsTxtFile {
22
- private robots;
23
- private proxyUrl?;
24
- private logger?;
22
+ #private;
25
23
  private constructor();
26
24
  /**
27
25
  * Determine the location of a robots.txt file for a URL and fetch it.
package/internals/robots.js CHANGED
@@ -20,13 +20,13 @@ import { Sitemap } from './sitemap.js';
20
20
  * ```
21
21
  */
22
22
  export class RobotsTxtFile {
23
- robots;
24
- proxyUrl;
25
- logger;
23
+ #robots;
24
+ #proxyUrl;
25
+ #logger;
26
26
  constructor(robots, proxyUrl, logger) {
27
- this.robots = robots;
28
- this.proxyUrl = proxyUrl;
29
- this.logger = logger;
27
+ this.#robots = robots;
28
+ this.#proxyUrl = proxyUrl;
29
+ this.#logger = logger;
30
30
  }
31
31
  /**
32
32
  * Determine the location of a robots.txt file for a URL and fetch it.
@@ -81,19 +81,19 @@ export class RobotsTxtFile {
81
81
  * @param [userAgent] relevant user agent, default to `*`
82
82
  */
83
83
  isAllowed(url, userAgent = '*') {
84
- return this.robots.isAllowed(url, userAgent) ?? true; // `undefined` means that there is no explicit rule for the requested URL - assume it's allowed
84
+ return this.#robots.isAllowed(url, userAgent) ?? true; // `undefined` means that there is no explicit rule for the requested URL - assume it's allowed
85
85
  }
86
86
  /**
87
87
  * Get URLs of sitemaps referenced in the robots file.
88
88
  */
89
89
  getSitemaps() {
90
- return this.robots.getSitemaps();
90
+ return this.#robots.getSitemaps();
91
91
  }
92
92
  /**
93
93
  * Parse all the sitemaps referenced in the robots file.
94
94
  */
95
95
  async parseSitemaps() {
96
- return Sitemap.load(this.robots.getSitemaps(), this.proxyUrl, { logger: this.logger });
96
+ return Sitemap.load(this.#robots.getSitemaps(), this.#proxyUrl, { logger: this.#logger });
97
97
  }
98
98
  /**
99
99
  * Get all URLs from all the sitemaps referenced in the robots file. A shorthand for `(await robots.parseSitemaps()).urls`.
package/internals/sitemap.js CHANGED
@@ -9,25 +9,25 @@ import MIMEType from 'whatwg-mimetype';
9
9
  import { mergeAsyncIterables } from './iterables.js';
10
10
  import { RobotsTxtFile } from './robots.js';
11
11
  class SitemapTxtParser extends Transform {
12
- decoder = new StringDecoder('utf8');
13
- buffer = '';
12
+ #decoder = new StringDecoder('utf8');
13
+ #buffer = '';
14
14
  constructor() {
15
15
  super({
16
16
  readableObjectMode: true,
17
17
  transform: (chunk, _encoding, callback) => {
18
- this.processBuffer(this.decoder.write(chunk), false);
18
+ this.processBuffer(this.#decoder.write(chunk), false);
19
19
  callback();
20
20
  },
21
21
  flush: (callback) => {
22
- this.processBuffer(this.decoder.end(), true);
22
+ this.processBuffer(this.#decoder.end(), true);
23
23
  callback();
24
24
  },
25
25
  });
26
26
  }
27
27
  processBuffer(input, finalize) {
28
- this.buffer += input;
29
- if (finalize || this.buffer.includes('\n')) {
30
- const parts = this.buffer
28
+ this.#buffer += input;
29
+ if (finalize || this.#buffer.includes('\n')) {
30
+ const parts = this.#buffer
31
31
  .split('\n')
32
32
  .map((part) => part.trim())
33
33
  .filter((part) => part.length > 0);
@@ -35,95 +35,95 @@ class SitemapTxtParser extends Transform {
35
35
  for (const url of parts) {
36
36
  this.push({ type: 'url', loc: url });
37
37
  }
38
- this.buffer = '';
38
+ this.#buffer = '';
39
39
  }
40
40
  else if (parts.length > 0) {
41
41
  for (const url of parts.slice(0, -1)) {
42
42
  this.push({ type: 'url', loc: url });
43
43
  }
44
- this.buffer = parts.at(-1);
44
+ this.#buffer = parts.at(-1);
45
45
  }
46
46
  }
47
47
  }
48
48
  }
49
49
  class SitemapXmlParser extends Transform {
50
- decoder = new StringDecoder('utf8');
51
- parser = new sax.SAXParser(true);
52
- rootTagName;
53
- currentTag = undefined;
54
- url = {};
50
+ #decoder = new StringDecoder('utf8');
51
+ #parser = new sax.SAXParser(true);
52
+ #rootTagName;
53
+ #currentTag = undefined;
54
+ #url = {};
55
55
  constructor() {
56
56
  super({
57
57
  readableObjectMode: true,
58
58
  transform: (chunk, _encoding, callback) => {
59
- this.parser.write(this.decoder.write(chunk));
59
+ this.#parser.write(this.#decoder.write(chunk));
60
60
  callback();
61
61
  },
62
62
  flush: (callback) => {
63
- const rest = this.decoder.end();
63
+ const rest = this.#decoder.end();
64
64
  if (rest.length > 0) {
65
- this.parser.write(rest);
65
+ this.#parser.write(rest);
66
66
  }
67
- this.parser.end();
67
+ this.#parser.end();
68
68
  callback();
69
69
  },
70
70
  });
71
- this.parser.onopentag = this.onOpenTag.bind(this);
72
- this.parser.onclosetag = this.onCloseTag.bind(this);
73
- this.parser.ontext = this.onText.bind(this);
74
- this.parser.oncdata = this.onText.bind(this);
75
- this.parser.onerror = this.destroy.bind(this);
71
+ this.#parser.onopentag = this.onOpenTag.bind(this);
72
+ this.#parser.onclosetag = this.onCloseTag.bind(this);
73
+ this.#parser.ontext = this.onText.bind(this);
74
+ this.#parser.oncdata = this.onText.bind(this);
75
+ this.#parser.onerror = this.destroy.bind(this);
76
76
  }
77
77
  onOpenTag(node) {
78
- if (this.rootTagName !== undefined) {
78
+ if (this.#rootTagName !== undefined) {
79
79
  if (node.name === 'loc' ||
80
80
  node.name === 'lastmod' ||
81
81
  node.name === 'priority' ||
82
82
  node.name === 'changefreq') {
83
- this.currentTag = node.name;
83
+ this.#currentTag = node.name;
84
84
  }
85
85
  }
86
86
  if (node.name === 'urlset') {
87
- this.rootTagName = 'urlset';
87
+ this.#rootTagName = 'urlset';
88
88
  }
89
89
  if (node.name === 'sitemapindex') {
90
- this.rootTagName = 'sitemapindex';
90
+ this.#rootTagName = 'sitemapindex';
91
91
  }
92
92
  }
93
93
  onCloseTag(name) {
94
94
  if (name === 'loc' || name === 'lastmod' || name === 'priority' || name === 'changefreq') {
95
- this.currentTag = undefined;
95
+ this.#currentTag = undefined;
96
96
  }
97
97
  if (name === 'url') {
98
- if (this.url.loc !== undefined) {
99
- this.push({ type: 'url', ...this.url, loc: this.url.loc });
98
+ if (this.#url.loc !== undefined) {
99
+ this.push({ type: 'url', ...this.#url, loc: this.#url.loc });
100
100
  }
101
- this.url = {};
101
+ this.#url = {};
102
102
  }
103
103
  }
104
104
  onText(text) {
105
- if (this.currentTag === 'loc') {
106
- if (this.rootTagName === 'sitemapindex') {
105
+ if (this.#currentTag === 'loc') {
106
+ if (this.#rootTagName === 'sitemapindex') {
107
107
  this.push({ type: 'sitemapUrl', url: text.trim() });
108
108
  }
109
- if (this.rootTagName === 'urlset') {
110
- this.url ??= {};
111
- this.url.loc = text.trim();
109
+ if (this.#rootTagName === 'urlset') {
110
+ this.#url ??= {};
111
+ this.#url.loc = text.trim();
112
112
  }
113
113
  }
114
114
  text = text.trim();
115
- if (this.currentTag === 'lastmod') {
115
+ if (this.#currentTag === 'lastmod') {
116
116
  const lastmod = new Date(text);
117
117
  if (!Number.isNaN(lastmod.getTime())) {
118
- this.url.lastmod = lastmod;
118
+ this.#url.lastmod = lastmod;
119
119
  }
120
120
  }
121
- if (this.currentTag === 'priority') {
122
- this.url.priority = Number(text);
121
+ if (this.#currentTag === 'priority') {
122
+ this.#url.priority = Number(text);
123
123
  }
124
- if (this.currentTag === 'changefreq') {
124
+ if (this.#currentTag === 'changefreq') {
125
125
  if (['always', 'hourly', 'daily', 'weekly', 'monthly', 'yearly', 'never'].includes(text)) {
126
- this.url.changefreq = text;
126
+ this.#url.changefreq = text;
127
127
  }
128
128
  }
129
129
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/utils",
3
- "version": "4.0.0-beta.105",
3
+ "version": "4.0.0-beta.107",
4
4
  "description": "A set of shared utilities that can be used by crawlers",
5
5
  "engines": {
6
6
  "node": ">=22.0.0"
@@ -42,8 +42,8 @@
42
42
  },
43
43
  "dependencies": {
44
44
  "@apify/ps-tree": "^1.2.0",
45
- "@crawlee/http-client": "4.0.0-beta.105",
46
- "@crawlee/types": "4.0.0-beta.105",
45
+ "@crawlee/http-client": "4.0.0-beta.107",
46
+ "@crawlee/types": "4.0.0-beta.107",
47
47
  "@types/sax": "^1.2.7",
48
48
  "cheerio": "^1.0.0",
49
49
  "domhandler": "^5.0.3",
@@ -61,5 +61,5 @@
61
61
  }
62
62
  }
63
63
  },
64
- "gitHead": "26073a822c7699ac487931383a39192bc3daae7a"
64
+ "gitHead": "5a3cb6242d0ae261df93c5e03cfebbb9b736e6b7"
65
65
  }