@crawlee/utils 3.17.1-beta.5 → 3.17.1-beta.51

package/internals/general.js

@@ -16,12 +16,12 @@ const promises_2 = require("node:timers/promises");
  * Default regular expression to match URLs in a string that may be plain text, JSON, CSV or other. It supports common URL characters
  * and does not support URLs containing commas or spaces. The URLs also may contain Unicode letters (not symbols).
  */
-exports.URL_NO_COMMAS_REGEX = /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+.~#?&/=()]*)?/giu;
+exports.URL_NO_COMMAS_REGEX = /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+.~#?&/=()'*]*)?/giu;
 /**
  * Regular expression that, in addition to the default regular expression `URL_NO_COMMAS_REGEX`, supports matching commas in URL path and query.
  * Note, however, that this may prevent parsing URLs from comma delimited lists, or the URLs may become malformed.
  */
-exports.URL_WITH_COMMAS_REGEX = /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+,.~#?&/=()]*)?/giu;
+exports.URL_WITH_COMMAS_REGEX = /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+,.~#?&/=()'*]*)?/giu;
 let isDockerPromiseCache;
 async function createIsDockerPromise() {
     const promise1 = promises_1.default

package/internals/open_graph_parser.js

@@ -258,32 +258,32 @@ const OPEN_GRAPH_PROPERTIES = [
         outputName: 'articleInfo',
         children: [
             {
-                name: 'music:published_time',
+                name: 'article:published_time',
                 outputName: 'publishedTime',
                 children: [],
             },
             {
-                name: 'music:modified_time',
+                name: 'article:modified_time',
                 outputName: 'modifiedTime',
                 children: [],
             },
             {
-                name: 'music:expiration_time',
+                name: 'article:expiration_time',
                 outputName: 'expirationTime',
                 children: [],
             },
             {
-                name: 'music:author',
+                name: 'article:author',
                 outputName: 'author',
                 children: [],
             },
             {
-                name: 'music:section',
+                name: 'article:section',
                 outputName: 'section',
                 children: [],
             },
             {
-                name: 'music:tag',
+                name: 'article:tag',
                 outputName: 'tag',
                 children: [],
             },

package/internals/sitemap.js

@@ -134,8 +134,10 @@ class SitemapXmlParser extends node_stream_1.Transform {
         if (name === 'loc' || name === 'lastmod' || name === 'priority' || name === 'changefreq') {
             this.currentTag = undefined;
         }
-        if (name === 'url' && this.url.loc !== undefined) {
-            this.push({ type: 'url', ...this.url, loc: this.url.loc });
+        if (name === 'url') {
+            if (this.url.loc !== undefined) {
+                this.push({ type: 'url', ...this.url, loc: this.url.loc });
+            }
             this.url = {};
         }
     }
@@ -151,7 +153,10 @@ class SitemapXmlParser extends node_stream_1.Transform {
         }
         text = text.trim();
         if (this.currentTag === 'lastmod') {
-            this.url.lastmod = new Date(text);
+            const lastmod = new Date(text);
+            if (!Number.isNaN(lastmod.getTime())) {
+                this.url.lastmod = lastmod;
+            }
         }
         if (this.currentTag === 'priority') {
             this.url.priority = Number(text);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/utils",
-    "version": "3.17.1-beta.5",
+    "version": "3.17.1-beta.51",
     "description": "A set of shared utilities that can be used by crawlers",
     "engines": {
         "node": ">=16.0.0"
@@ -49,7 +49,7 @@
     "dependencies": {
         "@apify/log": "^2.4.0",
         "@apify/ps-tree": "^1.2.0",
-        "@crawlee/types": "3.17.1-beta.5",
+        "@crawlee/types": "3.17.1-beta.51",
         "@types/sax": "^1.2.7",
         "cheerio": "1.0.0-rc.12",
         "file-type": "^21.3.1",
@@ -70,5 +70,5 @@
             }
         }
     },
-    "gitHead": "9479ce41601143fc01d56047f69d72ff54a33d1b"
+    "gitHead": "5185ab2c00a7fce4eaecb7e16c172b6077779a64"
 }