html-snapshots 5.7.2 → 5.9.0

package/lib/common/index.js

@@ -17,10 +17,8 @@
  */
 function ensure (options, must) {
   if (must) {
-    for (let prop in must) {
-      if (options[prop] === void 0 || options[prop] === null) {
-        options[prop] = must[prop];
-      }
+    for (const prop in must) {
+      options[prop] ??= must[prop];
     }
   }
   return options;
@@ -126,23 +124,25 @@ function prependMsgToErr (error, message, quoteInput) {
  * Simple response checker for remote files.
  * Expected use in robots.txt or sitemap.xml only.
  *
- * @param {IncomingMessage} res - The IncomingMessage response to check.
+ * @param {Response} res - The IncomingMessage response to check.
  * @param {Array} mediaTypes - array of acceptable content-type media type strings.
  * @returns {String} Error message, empty string (falsy) if OK.
  */
 function checkResponse (res, mediaTypes) {
-  let result = `status: '${res.statusCode}', GET failed.`;
+  let result = `status: '${res.status}', GET failed.`;
 
   mediaTypes = !Array.isArray(mediaTypes) ? [mediaTypes] : mediaTypes;
 
-  if (res.statusCode === 200) {
+  if (res.status === 200) {
+    const contentType = res.headers.get("Content-Type");
+
     // if content-type exists, and media type found then contentTypeOk
     const contentTypeOk =
-      res.headers["content-type"] &&
+      contentType &&
       // empty array and none found return true
       !mediaTypes.every(mediaType => {
         // flip -1 to 0 and NOT, so that true == NOT found, found stops loop w/false
-        return !~res.headers["content-type"].indexOf(mediaType);
+        return !~contentType.indexOf(mediaType);
       });
 
     result = contentTypeOk ? "" :
@@ -152,9 +152,43 @@ function checkResponse (res, mediaTypes) {
   return result;
 }
 
+/**
+ * Simple fetch with timeout and retries.
+ * Retries on throw (timeout) or 500+ response with backoff delay.
+ * 
+ * @param {String|Request} url - The fetch resource
+ * @param {Object} options - The fetch RequestInit object
+ * @param {Number} timeout - The request timeout in milliseconds
+ * @param {Number} retries - The retries on timeout or 500+ responses
+ * @param {Number} backoff - The initial backoff to wait between retries, backoff * 2 each iter
+ */
+async function simpleFetch (url, options = {}, timeout = 3000, retries = 3, backoff = 1000) {
+  try {
+    const response = await fetch(url, {
+      signal: AbortSignal.timeout(timeout),
+      ...options
+    });
+
+    if (!response.ok && response.status >= 500 && retries > 0) {
+      throw new Error(`Server error: ${response.status}`);
+    }
+
+    return response;
+  } catch (error) {
+    if (retries <= 0) throw error;
+    
+    console.warn(`Retrying... (${retries} attempts left)`);
+
+    await new Promise(resolve => setTimeout(resolve, backoff));
+
+    return simpleFetch(url, options, timeout, retries - 1, backoff * 2);
+  }
+}
+
 module.exports = {
   checkResponse,
   ensure,
+  get: (url, timeout) => simpleFetch(url, {}, timeout),
   head,
   isUrl,
   isFunction,

package/lib/common/sitemap/sitemap.js

@@ -178,23 +178,23 @@ function convert (options, buffer, next, callback) {
  * @return {Promise} resolves to data on completion.
  */
 async function getUrl (options, parseFn) {
-  const { default:got } = await import("got");
-  return got({
-    url: options.source,
-    responseType: "buffer",
-    timeout: {
-      request: options.timeout() // get the default timeout
-    }
-  }).then(res => {
-    let error = common.checkResponse(res, ["text/xml", "application/xml"]);
+  try {
+    const res = await common.get(options.source, options.timeout());
+    const error = common.checkResponse(res, ["text/xml", "application/xml"]);
     if (error) {
       throw new Error(error);
     }
+
+    const arrayBuffer = await res.arrayBuffer();
+
     const conv = util.promisify(convert);
-    return conv(options, res.body, parseFn);
-  }).catch(err => {
-    throw new Error(common.prependMsgToErr(err, options.source, true));
-  });
+    return conv(options, Buffer.from(arrayBuffer), parseFn);
+  }
+  catch (err) {
+    throw new Error(common.prependMsgToErr(err, options.source, true), {
+      cause: err
+    });
+  }
 }
 
 /**

package/lib/input-generators/robots.js

@@ -122,21 +122,19 @@ function processRobotsTxt (options, body) {
  * @returns {Promise} resolves to undefined.
  */
 async function getRobotsUrl (options) {
-  const { default:got } = await import("got");
-  return got({
-    url: options.source,
-    timeout: {
-      request: options.timeout()
-    }
-  }).then(res => {
+  try {
+    const res = await common.get(options.source, options.timeout());
     const error = common.checkResponse(res, "text/plain");
     if (error) {
       throw new Error(error);
     }
-    return processRobotsTxt(options, res.body.toString());
-  }).catch(err => {
-    throw new Error(common.prependMsgToErr(err, options.source, true));
-  });
+    return processRobotsTxt(options, await res.text());
+  }
+  catch (err) {
+    throw new Error(common.prependMsgToErr(err, options.source, true), {
+      cause: err
+    });
+  }
 }
 
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "html-snapshots",
-  "version": "5.7.2",
+  "version": "5.9.0",
   "author": {
     "name": "Alex Grant",
     "email": "alex@localnerve.com",
@@ -53,15 +53,14 @@
     "async": "3.2.6",
     "async-lock": "1.4.1",
     "combine-errors": "3.0.3",
-    "got": "^14.6.6",
     "phantomjs-prebuilt": "2.1.16",
-    "puppeteer": "^24.40.0",
+    "puppeteer": "^24.41.0",
     "xml2js": "0.6.2"
   },
   "devDependencies": {
     "eslint": "^10.2.0",
     "@eslint/js": "^10.0.1",
-    "globals": "^17.4.0",
+    "globals": "^17.5.0",
     "express": "^5.2.1",
     "mocha": "^11.7.5",
     "c8": "^11.0.0",
@@ -70,7 +69,7 @@
   },
   "overrides": {
     "request": "npm:@cypress/request@3.0.10",
-    "diff": "^8.0.4",
+    "diff": "^9.0.0",
     "glob": "^13.0.6",
     "serialize-javascript": "^7.0.5",
     "brace-expansion": "^5.0.5"