@crawlee/utils 4.0.0-beta.260 → 4.0.0-beta.262

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -31,26 +31,30 @@ function parseItem(context, element, ancestors) {
31
31
  }
32
32
  function collectPropertyElements(context, scope) {
33
33
  const elements = [];
34
- collectFromNodes(scope.children, elements);
34
+ // An `itemref` can point at an element already reached from the item's own subtree, or at one that
35
+ // contains another referenced element. The spec visits every element once, so each property counts once.
36
+ const visited = new Set();
37
+ collectFromNodes(scope.children, elements, visited);
35
38
  for (const id of uniqueTokens(scope.attribs.itemref)) {
36
39
  const referenced = (context.idIndex ??= indexIds(context.$)).get(id);
37
40
  if (referenced) {
38
- collectFromNodes([referenced], elements);
41
+ collectFromNodes([referenced], elements, visited);
39
42
  }
40
43
  }
41
44
  return elements;
42
45
  }
43
- function collectFromNodes(nodes, elements) {
46
+ function collectFromNodes(nodes, elements, visited) {
44
47
  for (const node of nodes) {
45
- if (!isTag(node)) {
48
+ if (!isTag(node) || visited.has(node)) {
46
49
  continue;
47
50
  }
51
+ visited.add(node);
48
52
  if ('itemprop' in node.attribs) {
49
53
  elements.push(node);
50
54
  }
51
55
  // A nested item owns everything below it, so its subtree is not part of the enclosing item.
52
56
  if (!('itemscope' in node.attribs)) {
53
- collectFromNodes(node.children, elements);
57
+ collectFromNodes(node.children, elements, visited);
54
58
  }
55
59
  }
56
60
  }
@@ -144,6 +144,10 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
144
144
  if (mimeType?.essence === 'text/plain' || url?.pathname.endsWith('.txt')) {
145
145
  return new SitemapTxtParser();
146
146
  }
147
+ // A name like `sitemap.gz` says nothing about what was compressed. Sitemaps are XML unless named otherwise.
148
+ if (mimeType?.essence === 'application/gzip') {
149
+ return SitemapXmlParser.create();
150
+ }
147
151
  throw new Error(`Unsupported sitemap content type (contentType = ${contentType}, url = ${url?.toString()})`);
148
152
  };
149
153
  while (sources.length > 0) {
@@ -235,44 +239,66 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
235
239
  // URL entries dropped by the enqueue strategy filter, reported in one warning per sitemap after
236
240
  // the loop (per-entry warnings could flood the log; individual drops are logged at debug level).
237
241
  let droppedUrlEntries = 0;
238
- for await (const item of items) {
239
- if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {
240
- if (nestedSitemapFilter && !nestedSitemapFilter(item.url)) {
241
- logger?.debug(`Skipping sitemap ${item.url} due to nestedSitemapFilter.`);
242
- continue;
242
+ const iterator = items[Symbol.asyncIterator]();
243
+ try {
244
+ while (true) {
245
+ let next;
246
+ try {
247
+ next = await iterator.next();
243
248
  }
244
- // Keep only nested sitemaps matching the strategy (and using http(s)) relative to the
245
- // parent. Raw string sources have no parent URL, so the check is skipped.
246
- if (source.type === 'url') {
247
- const { allowed, reason } = filterUrl(item.url, sitemapUrl, enqueueStrategy);
248
- if (!allowed) {
249
- logger?.warning(`Skipping nested sitemap ${item.url} (parent ${source.url}): ${reason}.`);
250
- continue;
249
+ catch (e) {
250
+ // Parsing failed: give up on this sitemap, keep the others.
251
+ // Raw sources already logged the error in their pipeline callback.
252
+ if (source.type !== 'raw') {
253
+ logger?.warning(`Malformed sitemap content: ${source.url} (${e})`);
251
254
  }
255
+ break;
252
256
  }
253
- sources.push({ type: 'url', url: item.url, depth: (source.depth ?? 0) + 1 });
254
- if (emitNestedSitemaps) {
255
- yield { loc: item.url, originSitemapUrl: null };
256
- }
257
- }
258
- if (item.type === 'url') {
259
- // Keep only URL entries that match the enqueue strategy relative to the parent (see above).
260
- if (source.type === 'url') {
261
- const { allowed, reason } = filterUrl(item.loc, sitemapUrl, enqueueStrategy);
262
- if (!allowed) {
263
- droppedUrlEntries++;
264
- logger?.debug(`Skipping sitemap URL ${item.loc} (parent ${source.url}): ${reason}.`);
257
+ if (next.done)
258
+ break;
259
+ const item = next.value;
260
+ if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {
261
+ if (nestedSitemapFilter && !nestedSitemapFilter(item.url)) {
262
+ logger?.debug(`Skipping sitemap ${item.url} due to nestedSitemapFilter.`);
265
263
  continue;
266
264
  }
265
+ // Keep only nested sitemaps matching the strategy (and using http(s)) relative to the
266
+ // parent. Raw string sources have no parent URL, so the check is skipped.
267
+ if (source.type === 'url') {
268
+ const { allowed, reason } = filterUrl(item.url, sitemapUrl, enqueueStrategy);
269
+ if (!allowed) {
270
+ logger?.warning(`Skipping nested sitemap ${item.url} (parent ${source.url}): ${reason}.`);
271
+ continue;
272
+ }
273
+ }
274
+ sources.push({ type: 'url', url: item.url, depth: (source.depth ?? 0) + 1 });
275
+ if (emitNestedSitemaps) {
276
+ yield { loc: item.url, originSitemapUrl: null };
277
+ }
278
+ }
279
+ if (item.type === 'url') {
280
+ // Keep only URL entries that match the enqueue strategy relative to the parent (see above).
281
+ if (source.type === 'url') {
282
+ const { allowed, reason } = filterUrl(item.loc, sitemapUrl, enqueueStrategy);
283
+ if (!allowed) {
284
+ droppedUrlEntries++;
285
+ logger?.debug(`Skipping sitemap URL ${item.loc} (parent ${source.url}): ${reason}.`);
286
+ continue;
287
+ }
288
+ }
289
+ yield {
290
+ ...item,
291
+ originSitemapUrl: source.type === 'url'
292
+ ? source.url
293
+ : `raw://${createHash('sha256').update(source.content).digest('base64')}`,
294
+ };
267
295
  }
268
- yield {
269
- ...item,
270
- originSitemapUrl: source.type === 'url'
271
- ? source.url
272
- : `raw://${createHash('sha256').update(source.content).digest('base64')}`,
273
- };
274
296
  }
275
297
  }
298
+ finally {
299
+ // Mirrors `for await` cleanup: destroys the stream when we stop early or the consumer bails out.
300
+ await iterator.return?.();
301
+ }
276
302
  if (droppedUrlEntries > 0 && source.type === 'url') {
277
303
  logger?.warning(`Skipped ${droppedUrlEntries} URL(s) from sitemap ${source.url} not matching enqueue strategy '${enqueueStrategy}' (or using a non-http(s) scheme). Enable debug logs to see each skipped URL.`);
278
304
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/utils",
3
- "version": "4.0.0-beta.260",
3
+ "version": "4.0.0-beta.262",
4
4
  "description": "A set of shared utilities that can be used by crawlers",
5
5
  "engines": {
6
6
  "node": ">=22.13.0"
@@ -43,8 +43,8 @@
43
43
  },
44
44
  "dependencies": {
45
45
  "@apify/validations": "^1.1.2",
46
- "@crawlee/http-client": "4.0.0-beta.260",
47
- "@crawlee/types": "4.0.0-beta.260",
46
+ "@crawlee/http-client": "4.0.0-beta.262",
47
+ "@crawlee/types": "4.0.0-beta.262",
48
48
  "@types/sax": "^1.2.7",
49
49
  "cheerio": "^1.2.0",
50
50
  "domhandler": "^6.0.1",
@@ -63,5 +63,5 @@
63
63
  }
64
64
  }
65
65
  },
66
- "gitHead": "c96c87d09eb5df1bcdf6365563ea99739f612a35"
66
+ "gitHead": "56d3158b87fd8f83d195634092f6fbfe3b0e4616"
67
67
  }