@crawlee/utils 3.17.1-beta.32 → 3.17.1-beta.34

package/internals/general.js

@@ -16,12 +16,12 @@ const promises_2 = require("node:timers/promises");
  * Default regular expression to match URLs in a string that may be plain text, JSON, CSV or other. It supports common URL characters
  * and does not support URLs containing commas or spaces. The URLs also may contain Unicode letters (not symbols).
  */
-exports.URL_NO_COMMAS_REGEX = /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+.~#?&/=()]*)?/giu;
+exports.URL_NO_COMMAS_REGEX = /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+.~#?&/=()'*]*)?/giu;
 /**
  * Regular expression that, in addition to the default regular expression `URL_NO_COMMAS_REGEX`, supports matching commas in URL path and query.
  * Note, however, that this may prevent parsing URLs from comma delimited lists, or the URLs may become malformed.
  */
-exports.URL_WITH_COMMAS_REGEX = /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+,.~#?&/=()]*)?/giu;
+exports.URL_WITH_COMMAS_REGEX = /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+,.~#?&/=()'*]*)?/giu;
 let isDockerPromiseCache;
 async function createIsDockerPromise() {
     const promise1 = promises_1.default

package/internals/sitemap.js

@@ -134,8 +134,10 @@ class SitemapXmlParser extends node_stream_1.Transform {
         if (name === 'loc' || name === 'lastmod' || name === 'priority' || name === 'changefreq') {
             this.currentTag = undefined;
         }
-        if (name === 'url' && this.url.loc !== undefined) {
-            this.push({ type: 'url', ...this.url, loc: this.url.loc });
+        if (name === 'url') {
+            if (this.url.loc !== undefined) {
+                this.push({ type: 'url', ...this.url, loc: this.url.loc });
+            }
             this.url = {};
         }
     }
@@ -151,7 +153,10 @@ class SitemapXmlParser extends node_stream_1.Transform {
         }
         text = text.trim();
         if (this.currentTag === 'lastmod') {
-            this.url.lastmod = new Date(text);
+            const lastmod = new Date(text);
+            if (!Number.isNaN(lastmod.getTime())) {
+                this.url.lastmod = lastmod;
+            }
         }
         if (this.currentTag === 'priority') {
             this.url.priority = Number(text);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/utils",
-    "version": "3.17.1-beta.32",
+    "version": "3.17.1-beta.34",
     "description": "A set of shared utilities that can be used by crawlers",
     "engines": {
         "node": ">=16.0.0"
@@ -49,7 +49,7 @@
     "dependencies": {
         "@apify/log": "^2.4.0",
         "@apify/ps-tree": "^1.2.0",
-        "@crawlee/types": "3.17.1-beta.32",
+        "@crawlee/types": "3.17.1-beta.34",
         "@types/sax": "^1.2.7",
         "cheerio": "1.0.0-rc.12",
         "file-type": "^21.3.1",
@@ -70,5 +70,5 @@
             }
         }
     },
-    "gitHead": "d8bbdbb6b21c1171f8c4f8593675f2aad98ca4fa"
+    "gitHead": "bac9190dce2ecb2ab994c749839c7a89d3c6ac3d"
 }