@util.js/node-http 0.41.8 → 0.42.1

package/index.js

@@ -1,4 +1,3 @@
-"use strict";
+import Urls from "./lib/Urls.js";
 
-const Urls = require("./lib/Urls");
-module.exports = new Urls();
+export default Urls;

package/lib/Urls.js

@@ -1,154 +1,419 @@
-"use strict";
+import https from "node:https";
+import http from "node:http";
+import { pipeline } from "node:stream/promises";
+import Files from "@util.js/node-files";
+
+// These are global variables for Chromium version management.
+let latestChromiumAndroidStableVersion;
+let latestChromiumAndroidStableVersionLocked = false;
+let googleBotUserAgentHeaderLogged = false;
+
+/**
+ * Determine timeout value in milliseconds from the parameters object.
+ */
+function determineTimeout(params) {
+  if (params == null) return -1;
+  const timeoutVal = params?.timeoutMs;
+  const parsedInt = Number.parseInt(timeoutVal, 10);
+  if (timeoutVal != null && !Number.isNaN(parsedInt) && parsedInt >= 0) {
+    return parsedInt;
+  }
+  return -1;
+}
+
+/**
+ * Fetch the latest Chromium Android stable version from the API.
+ */
+async function fetchLatestChromiumAndroidStableVersion() {
+  const CHANNEL = "Stable";
+  const url = `https://chromiumdash.appspot.com/fetch_releases?platform=Android&channel=${CHANNEL}&num=1`;
+
+  const response = await fetch(url);
+  if (!response.ok)
+    throw new Error(
+      `Fetching ${url} failed with HTTP status ${response.status}.`,
+    );
+
+  const data = await response.json();
+  if (!data || data.length === 0)
+    throw new Error("The Chromium Dash API returned an empty response.");
+
+  const latest = data[0];
+  if (!latest || !latest.version)
+    throw new Error(
+      "The version field is missing in the Chromium Dash API response.",
+    );
+  return latest.version;
+}
+
+/**
+ * Get the Google Bot user agent header with the latest Chromium version.
+ */
+async function getGoogleBotUserAgentHeader(logger) {
+  // https://chromiumdash.appspot.com/releases?platform=Android
+  // https://github.com/wasi-master/13ft/blob/main/app/portable.py
+  // https://web.archive.org/web/20260811103458/https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers#googlebot-news#googlebot-news
+
+  const latestChromiumAndroidStableVersion =
+    await getLatestChromiumAndroidStableVersion(logger);
+  const userAgent = `Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${latestChromiumAndroidStableVersion} Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)`;
+
+  if (!googleBotUserAgentHeaderLogged) {
+    googleBotUserAgentHeaderLogged = true;
+    logger?.error?.("Using the following user agent for HTTP requests: ");
+    logger?.error?.(userAgent);
+    logger?.error?.();
+  }
+
+  return userAgent;
+}
+
+/**
+ * Get the latest Chromium Android stable version with fallback support.
+ */
+async function getLatestChromiumAndroidStableVersion(logger) {
+  while (latestChromiumAndroidStableVersionLocked) {
+    await new Promise((resolve) => setTimeout(resolve, 10));
+  }
+  latestChromiumAndroidStableVersionLocked = true;
+  try {
+    if (latestChromiumAndroidStableVersion)
+      return latestChromiumAndroidStableVersion;
+
+    try {
+      latestChromiumAndroidStableVersion =
+        await fetchLatestChromiumAndroidStableVersion();
+    } catch (fetchErr) {
+      const fallbackVersion = "152.0.7977.42";
+      logger?.error?.(
+        `Fetching the latest Chromium version failed: ${fetchErr.message}`,
+      );
+      logger?.error?.(
+        `Falling back to hardcoded version '${fallbackVersion}'...`,
+      );
+      latestChromiumAndroidStableVersion = fallbackVersion;
+    }
+
+    logger?.error?.();
+    logger?.error?.(
+      `The latest Chromium Android stable version is '${latestChromiumAndroidStableVersion}'.`,
+    );
+    logger?.error?.();
+    return latestChromiumAndroidStableVersion;
+  } finally {
+    latestChromiumAndroidStableVersionLocked = false;
+  }
+}
+
+/**
+ * Normalize the input into a valid URL object.
+ */
+function parseUrl(url) {
+  try {
+    return typeof url === "string" ? new URL(url) : url;
+  } catch (err) {
+    throw new TypeError(`The provided URL "${url}" is invalid.`);
+  }
+}
 
 class Urls {
-  decodeURI() {
-    return decodeURI(...arguments);
-  }
-
-  decodeURIComponent() {
-    return decodeURIComponent(...arguments);
-  }
-
-  // http://stackoverflow.com/questions/11944932/how-to-download-a-file-with-node-js
-  download(url, destination, callbackOrOptions, callback) {
-    if (typeof arguments[arguments.length - 1] !== "function")
-      return promises().callCallback(this, this.download, ...arguments);
-    const cb = arguments[arguments.length - 1];
-    const parsedUrl = URL ? new URL(url) : urlUtil().parse(url);
-    const file = files().createWriteStream(destination);
-    const options = callbackOrOptions
-      ? typeof callbackOrOptions === "function"
-        ? {}
-        : callbackOrOptions
-      : {};
-    const request = getProtocolObject(parsedUrl.protocol)
-      .get(url, options, (response) => {
-        response.pipe(file);
-        file.on("finish", () => file.close(cb));
-      })
-      .on("error", (error) => {
-        files().unlink(destination, (unlinkError) => {
-          cb(unlinkError || error);
-        });
+  /**
+   * Decode a Uniform Resource Identifier (URI).
+   */
+  static decodeURI(...args) {
+    return decodeURI(...args);
+  }
+
+  /**
+   * Decode a Uniform Resource Identifier (URI) component.
+   */
+  static decodeURIComponent(...args) {
+    return decodeURIComponent(...args);
+  }
+
+  /**
+   * Encode a Uniform Resource Identifier (URI).
+   */
+  static encodeURI(...args) {
+    return encodeURI(...args);
+  }
+
+  /**
+   * Encode a Uniform Resource Identifier (URI) component.
+   */
+  static encodeURIComponent(...args) {
+    return encodeURIComponent(...args);
+  }
+
+  /**
+   * Get HTTP headers for the given URL using a HEAD request.
+   * http://stackoverflow.com/questions/5922842/getting-http-headers-with-node-js
+   * http://stackoverflow.com/a/6001507
+   */
+  static async headers(params) {
+    const timeoutMs = determineTimeout(params);
+    const {
+      headers: customHeaders = {},
+      logger,
+      url,
+      useGoogleBotUserAgent = false,
+    } = params || {};
+
+    const urlObj = parseUrl(url);
+    const p = urlObj.protocol === "http:" ? http : https;
+    let headers = { ...customHeaders };
+
+    if (useGoogleBotUserAgent && !headers["User-Agent"]) {
+      headers["User-Agent"] = await getGoogleBotUserAgentHeader(logger);
+    }
+
+    const reqOptions = {
+      ...params,
+      timeout: timeoutMs !== -1 ? timeoutMs : undefined,
+      headers,
+      hostname: urlObj.hostname,
+      method: "HEAD",
+      path: urlObj.pathname + urlObj.search,
+      port: urlObj.port || (urlObj.protocol === "https:" ? 443 : 80),
+    };
+
+    return new Promise((resolve, reject) => {
+      const req = p.request(reqOptions, (res) => {
+        res.destroy();
+        resolve(res.headers);
       });
-    const timeout = determineTimeout(callbackOrOptions);
-    if (timeout !== -1) {
-      // https://stackoverflow.com/a/12783062
-      request.on("socket", (socket) => {
-        socket.setTimeout(timeout);
-        socket.on("timeout", () => {
-          request.abort();
+
+      req.on("error", reject);
+
+      if (timeoutMs !== -1) {
+        // https://stackoverflow.com/a/12783062
+        req.on("socket", (socket) => {
+          socket.setTimeout(timeoutMs);
+          socket.on("timeout", () => {
+            req.destroy(new Error("The request timed out."));
+          });
         });
-      });
-    }
+      }
+
+      req.end();
+    });
   }
 
-  encodeURI() {
-    return encodeURI(...arguments);
+  /**
+   * Convert the given URL content to a buffer.
+   */
+  static async urlToBuffer(params) {
+    const timeoutMs = determineTimeout(params);
+    const { logger, url, useGoogleBotUserAgent = false } = params || {};
+    const chunks = await this.urlToData({
+      logger,
+      timeoutMs,
+      url,
+      useGoogleBotUserAgent,
+    });
+    return Buffer.concat(chunks);
   }
 
-  encodeURIComponent() {
-    return encodeURIComponent(...arguments);
+  /**
+   * Get data from the given URL as chunks.
+   */
+  static async urlToData(params) {
+    const timeoutMs = determineTimeout(params);
+    const {
+      headers,
+      logger,
+      url,
+      useGoogleBotUserAgent = false,
+    } = params || {};
+
+    const readable = await this.urlToReadStream({
+      headers,
+      logger,
+      timeoutMs,
+      url,
+      useGoogleBotUserAgent,
+    });
+
+    const chunks = [];
+    for await (const chunk of readable) {
+      chunks.push(chunk);
+    }
+    return chunks;
   }
 
-  // http://stackoverflow.com/questions/5922842/getting-http-headers-with-node-js
-  // http://stackoverflow.com/a/6001507
-  headers(url, callbackOrOptions, callback) {
-    if (typeof arguments[arguments.length - 1] !== "function")
-      return promises().callCallback(this, this.headers, ...arguments);
-    const cb = arguments[arguments.length - 1];
-    const parsedUrl = URL ? new URL(url) : urlUtil().parse(url);
-    const port = parsedUrl.port
-      ? parsedUrl.port
-      : parsedUrl.protocol === "https:"
-        ? 443
-        : 80;
+  /**
+   * Download the given URL to the given local file.
+   */
+  static async urlToFile(params) {
+    const timeoutMs = determineTimeout(params);
+    const {
+      file,
+      logger,
+      maxRedirects,
+      url,
+      useGoogleBotUserAgent = false,
+    } = params || {};
 
-    let options = Object.assign(
-      {},
-      callbackOrOptions
-        ? typeof callbackOrOptions === "function"
-          ? {}
-          : callbackOrOptions
-        : {},
-    );
-    options = Object.assign(options, {
-      host: parsedUrl.host,
-      method: "HEAD",
-      path: parsedUrl.path,
-      port,
+    const readable = await this.urlToReadStream({
+      logger,
+      maxRedirects,
+      timeoutMs,
+      url,
+      useGoogleBotUserAgent,
     });
-    let cbCalled = false;
-    const request = getProtocolObject(parsedUrl.protocol)
-      .request(options, (res) => {
-        res.destroy();
-        if (!cbCalled) {
-          cbCalled = true;
-          cb(null, res.headers);
-        }
-      })
-      .on("error", (error) => {
-        if (!cbCalled) {
-          cbCalled = true;
-          cb(error);
+
+    const writable = Files.createWriteStream(file);
+    await pipeline(readable, writable);
+  }
+
+  /**
+   * Get the HTTP stream for the given URL with redirect and timeout support.
+   */
+  static async urlToReadStream(params) {
+    const timeoutMs = determineTimeout(params);
+    const {
+      headers: customHeaders = {},
+      logger,
+      maxRedirects = 0,
+      url,
+      useGoogleBotUserAgent = false,
+    } = params || {};
+
+    const urlObj = parseUrl(url);
+    const p = urlObj.protocol === "http:" ? http : https;
+    let headers = { ...customHeaders };
+
+    if (useGoogleBotUserAgent && !headers["User-Agent"]) {
+      headers["User-Agent"] = await getGoogleBotUserAgentHeader(logger);
+    }
+
+    return new Promise((resolve, reject) => {
+      const req = p.get(urlObj, { headers }, (res) => {
+        const { statusCode } = res;
+
+        if (statusCode !== 200) {
+          if (statusCode >= 300 && statusCode < 400 && res.headers.location) {
+            if (maxRedirects > 0) {
+              const newUrl = new URL(res.headers.location, urlObj);
+              res.resume(); // Free memory.
+              Urls.urlToReadStream({
+                headers,
+                logger,
+                maxRedirects: maxRedirects - 1,
+                timeoutMs,
+                url: newUrl,
+                useGoogleBotUserAgent,
+              })
+                .then(resolve)
+                .catch(reject);
+              return;
+            }
+          }
+
+          res.resume();
+          const error = new Error(
+            `The request for ${urlObj.href} failed with status code ${statusCode}.`,
+          );
+          error.headers = res.headers;
+          error.statusCode = statusCode;
+          error.url = urlObj.href;
+          return reject(error);
         }
+
+        resolve(res);
       });
-    const timeout = determineTimeout(callbackOrOptions);
-    if (timeout !== -1) {
-      // https://stackoverflow.com/a/12783062
-      request.on("socket", (socket) => {
-        socket.setTimeout(timeout);
-        socket.on("timeout", () => {
-          request.destroy(new Error("The request timed out."));
+
+      req.on("error", reject);
+
+      if (timeoutMs !== -1) {
+        // https://stackoverflow.com/a/12783062
+        req.on("socket", (socket) => {
+          socket.setTimeout(timeoutMs);
+          socket.on("timeout", () => {
+            req.destroy(new Error("The request timed out."));
+          });
         });
-      });
-    }
-    request.end();
+      }
+    });
   }
-}
 
-module.exports = Urls;
+  /**
+   * Proxy the HTTP request for the given URL to the given web response.
+   */
+  static async urlToResponse(params) {
+    const timeoutMs = determineTimeout(params);
+    const {
+      logger,
+      response,
+      url,
+      useGoogleBotUserAgent = false,
+    } = params || {};
 
-function determineTimeout(params) {
-  if (!params) return -1;
-  const parsedInt = Number.parseInt(params.timeout);
-  if (params.timeout != null && !Number.isNaN(parsedInt) && parsedInt >= 0)
-    return parsedInt;
-  return -1;
-}
+    const readable = await Urls.urlToReadStream({
+      logger,
+      timeoutMs,
+      url,
+      useGoogleBotUserAgent,
+    });
 
-const dependencies = {};
-function get(dependency) {
-  return (
-    dependencies[dependency] || (dependencies[dependency] = require(dependency))
-  );
-}
+    response.set(readable.headers);
+    await pipeline(readable, response);
+  }
 
-function files() {
-  return get("@util.js/node-files");
-}
-function http() {
-  return get("http");
-}
-function https() {
-  return get("https");
-}
-function promises() {
-  return get("@util.js/promises");
-}
-function urlUtil() {
-  return get("url");
-}
+  /**
+   * Convert the given URL content to a string, auto-detecting its charset from Content-Type headers.
+   */
+  static async urlToString(params) {
+    const timeoutMs = determineTimeout(params);
+    const {
+      encoding: explicitEncoding,
+      headers,
+      logger,
+      url,
+      useGoogleBotUserAgent = false,
+    } = params || {};
 
-function getProtocolObject(protocolStr) {
-  switch (protocolStr) {
-    case "https:":
-      return https();
-    case "http:":
-      return http();
-    default:
-      throw new TypeError(
-        'The protocol must be "https:" or "http:", not "' + protocolStr + '"',
-      );
+    const readable = await this.urlToReadStream({
+      headers,
+      logger,
+      timeoutMs,
+      url,
+      useGoogleBotUserAgent,
+    });
+
+    let encoding = explicitEncoding;
+
+    if (!encoding) {
+      const contentType = readable.headers?.["content-type"];
+      if (contentType) {
+        const match = contentType.match(/charset=([^;]+)/i);
+        if (match) {
+          encoding = match[1].trim().replace(/['"]/g, "").toLowerCase();
+        }
+      }
+    }
+
+    // Map common charset names to Node.js Buffer encodings.
+    if (encoding === "iso-8859-1") {
+      encoding = "latin1";
+    } else if (encoding === "utf-8") {
+      encoding = "utf8";
+    }
+
+    const chunks = [];
+    for await (const chunk of readable) {
+      chunks.push(chunk);
+    }
+
+    const buffer = Buffer.concat(chunks);
+
+    try {
+      return buffer.toString(encoding || "utf8");
+    } catch (err) {
+      // Fall back to utf8 if Node.js does not recognize the encoding.
+      return buffer.toString("utf8");
+    }
   }
 }
+
+export default Urls;

package/package.json

@@ -1,10 +1,13 @@
 {
   "author": "Chris Topher <c@creemama.com>",
   "dependencies": {
-    "@util.js/node-files": "0.41.8",
-    "@util.js/promises": "0.41.8"
+    "@util.js/node-files": "0.42.1",
+    "@util.js/promises": "0.42.1"
   },
   "description": "JavaScript utility methods for URLs",
+  "exports": {
+    ".": "./index.js"
+  },
   "files": [
     "index.js",
     "lib"
@@ -19,6 +22,7 @@
     "utiljs"
   ],
   "license": "MIT",
+  "main": "./index.js",
   "name": "@util.js/node-http",
   "publishConfig": {
     "access": "public"
@@ -31,5 +35,6 @@
     "type": "git",
     "url": "https://github.com/creemama/utiljs/tree/master/packages/utiljs-node-http"
   },
-  "version": "0.41.8"
+  "type": "module",
+  "version": "0.42.1"
 }