@automattic/vip 4.0.5 → 4.0.6

package/dist/lib/client-file-uploader.js

@@ -4,6 +4,7 @@ exports.__esModule = true;
 exports.MULTIPART_THRESHOLD = exports.COMPRESS_THRESHOLD = void 0;
 exports.checkFileAccess = checkFileAccess;
 exports.detectCompressedMimeType = detectCompressedMimeType;
+exports.fetchWithRetry = fetchWithRetry;
 exports.getFileHash = void 0;
 exports.getFileMeta = getFileMeta;
 exports.getFileSize = getFileSize;
@@ -20,6 +21,7 @@ var _debug = _interopRequireDefault(require("debug"));
 var _fs = require("fs");
 var _promises = require("node:fs/promises");
 var _promises2 = require("node:stream/promises");
+var _nodeTimers = require("node:timers");
 var _promises3 = require("node:timers/promises");
 var _os = _interopRequireDefault(require("os"));
 var _path = _interopRequireDefault(require("path"));
@@ -41,13 +43,34 @@ function parseEtagHeader(etag) {
   }
   return normalizedEtag.replace(/^"(.*)"$/u, '$1');
 }
-async function fetchWithRetry(input, init, retries = 3) {
-  for (let attempt = 0; attempt <= retries; attempt++) {
+/**
+ * Wraps `fetch` with exponential-backoff retries.
+ *
+ * A request body that is a Node stream can only be consumed once. If the body
+ * is a stream and we retried with the same `init`, the second attempt would
+ * pass an already-consumed (disturbed/locked) stream to `fetch`, which throws
+ * the misleading `Response body object should not be disturbed or locked` error
+ * and masks the original network failure.
+ *
+ * To retry safely, callers that send a stream body should pass `createBody`,
+ * which is invoked to produce a fresh body for every attempt. If a stream body
+ * is supplied without a `createBody` factory, the request is attempted only
+ * once so the underlying error surfaces instead of the misleading undici one.
+ */
+async function fetchWithRetry(input, init = {}, retries = 3, createBody) {
+  const bodyIsStream = typeof init.body?.pipe === 'function';
+  // Only retry when we can hand `fetch` a fresh, replayable body each attempt.
+  const maxAttempts = createBody || !bodyIsStream ? retries : 0;
+  for (let attempt = 0; attempt <= maxAttempts; attempt++) {
+    const requestInit = createBody ? {
+      ...init,
+      body: createBody()
+    } : init;
     try {
       // eslint-disable-next-line no-await-in-loop
-      return await (0, _undici.fetch)(input, init);
+      return await (0, _undici.fetch)(input, requestInit);
     } catch (err) {
-      if (attempt === retries) {
+      if (attempt === maxAttempts) {
         throw err;
       }
       // eslint-disable-next-line no-await-in-loop
@@ -220,20 +243,28 @@ async function uploadUsingPutObject({
     ...fetchOptions.headers,
     'Content-Length': `${fileSize}` // This has to be a string
   };
-  let readBytes = 0;
-  const progressPassThrough = new _stream.PassThrough();
-  progressPassThrough.on('data', data => {
-    readBytes += data.length;
-    const percentage = `${Math.floor(100 * readBytes / fileSize)}%`;
-    debug(percentage);
-    if (typeof progressCallback === 'function') {
-      progressCallback(percentage);
+
+  // Build a fresh request body for every attempt. The upload streams the file
+  // from disk, and a stream can only be consumed once; recreating it per
+  // attempt lets `fetchWithRetry` retry without reusing a consumed stream
+  // (which would otherwise throw `... should not be disturbed or locked`).
+  const createBody = () => {
+    if (fileContent) {
+      return fileContent;
     }
-  });
-  const response = await fetchWithRetry(presignedRequest.url, {
-    ...fetchOptions,
-    body: fileContent ?? (0, _fs.createReadStream)(fileName).pipe(progressPassThrough)
-  });
+    let readBytes = 0;
+    const progressPassThrough = new _stream.PassThrough();
+    progressPassThrough.on('data', data => {
+      readBytes += data.length;
+      const percentage = `${Math.floor(100 * readBytes / fileSize)}%`;
+      debug(percentage);
+      if (typeof progressCallback === 'function') {
+        progressCallback(percentage);
+      }
+    });
+    return (0, _fs.createReadStream)(fileName).pipe(progressPassThrough);
+  };
+  const response = await fetchWithRetry(presignedRequest.url, fetchOptions, 3, createBody);
   if (response.status === 200) {
     return 'ok';
   }
@@ -428,18 +459,19 @@ async function uploadParts({
   progressCallback
 }) {
   let uploadsInProgress = 0;
-  let totalBytesRead = 0;
   const partPercentages = new Array(parts.length).fill(0);
+  const partBytesRead = new Array(parts.length).fill(0);
   const readyForPartUpload = () => new Promise(resolve => {
-    const canDoInterval = setInterval(() => {
+    const canDoInterval = (0, _nodeTimers.setInterval)(() => {
       if (uploadsInProgress < MAX_CONCURRENT_PART_UPLOADS) {
         uploadsInProgress++;
-        clearInterval(canDoInterval);
+        (0, _nodeTimers.clearInterval)(canDoInterval);
         resolve();
       }
     }, 300);
   });
   const updateProgress = () => {
+    const totalBytesRead = partBytesRead.reduce((sum, bytes) => sum + bytes, 0);
     const percentage = `${Math.floor(100 * totalBytesRead / fileMeta.fileSize)}%`;
     if (typeof progressCallback === 'function') {
       progressCallback(percentage);
@@ -451,32 +483,46 @@ async function uploadParts({
       return `Part # ${index}: ${partPercentage}% of ${(partSize / _fileSize.MB_IN_BYTES).toFixed(2)}MB`;
     }).join('\n') + `\n\nOverall Progress: ${percentage}% of ${(fileMeta.fileSize / _fileSize.MB_IN_BYTES).toFixed(2)}MB`);
   };
-  const updateProgressInterval = setInterval(updateProgress, 500);
+  const updateProgressInterval = (0, _nodeTimers.setInterval)(updateProgress, 500);
   const allDone = await Promise.all(parts.map(async part => {
     const {
       index,
-      partSize
+      partSize,
+      start,
+      end
     } = part;
-    const progressPassThrough = new _stream.PassThrough();
-    let partBytesRead = 0;
-    progressPassThrough.on('data', data => {
-      totalBytesRead += data.length;
-      partBytesRead += data.length;
-      partPercentages[index] = Math.floor(100 * partBytesRead / partSize);
-    });
+
+    // Build a fresh request body for every attempt. A stream can only be
+    // consumed once, so recreating the ranged read stream (and its progress
+    // PassThrough) per attempt lets `fetchWithRetry` retry without reusing a
+    // disturbed/locked stream. Reset this part's progress counters so a retry
+    // re-streams the part without double-counting bytes.
+    const createBody = () => {
+      partBytesRead[index] = 0;
+      partPercentages[index] = 0;
+      const progressPassThrough = new _stream.PassThrough();
+      progressPassThrough.on('data', data => {
+        partBytesRead[index] += data.length;
+        partPercentages[index] = Math.floor(100 * partBytesRead[index] / partSize);
+      });
+      return (0, _fs.createReadStream)(fileMeta.fileName, {
+        start,
+        end
+      }).pipe(progressPassThrough);
+    };
     await readyForPartUpload();
     const uploadResult = await uploadPart({
       app,
       env,
       fileMeta,
       part,
-      progressPassThrough,
+      createBody,
       uploadId
     });
     uploadsInProgress--;
     return uploadResult;
   }));
-  clearInterval(updateProgressInterval);
+  (0, _nodeTimers.clearInterval)(updateProgressInterval);
   updateProgress();
   return allDone;
 }
@@ -484,18 +530,15 @@ async function uploadPart({
   app,
   env,
   fileMeta: {
-    basename,
-    fileName
+    basename
   },
   part,
-  progressPassThrough,
+  createBody,
   uploadId
 }) {
   const {
-    end,
     index,
-    partSize,
-    start
+    partSize
   } = part;
   const s3PartNumber = index + 1; // S3 multipart is indexed from 1
 
@@ -521,11 +564,7 @@ async function uploadPart({
        *   ...so it may not be feasible to include with presigned requests.
        */
     };
-    fetchOptions.body = (0, _fs.createReadStream)(fileName, {
-      start,
-      end
-    }).pipe(progressPassThrough);
-    const fetchResponse = await fetchWithRetry(partUploadRequestData.url, fetchOptions);
+    const fetchResponse = await fetchWithRetry(partUploadRequestData.url, fetchOptions, 3, createBody);
     if (fetchResponse.status === 200) {
       const etag = fetchResponse.headers.get('etag');
       if (!etag) {

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "@automattic/vip",
-  "version": "4.0.5",
+  "version": "4.0.6",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@automattic/vip",
-      "version": "4.0.5",
+      "version": "4.0.6",
       "hasInstallScript": true,
       "license": "MIT",
       "dependencies": {
@@ -28,13 +28,13 @@
         "graphql": "17.0.1",
         "graphql-tag": "2.12.7",
         "https-proxy-agent": "^9.0.0",
-        "js-yaml": "^4.1.0",
+        "js-yaml": "^5.0.0",
         "jwt-decode": "4.0.0",
         "lando": "github:automattic/lando-cli#78d382fcf40357c4a4cebe4a3cfaef032eab743a",
         "node-stream-zip": "1.15.0",
         "open": "^11.0.0",
         "proxy-from-env": "^2.0.0",
-        "semver": "7.8.4",
+        "semver": "7.8.5",
         "shelljs": "^0.10.0",
         "socket.io-client": "^4.5.3",
         "socket.io-stream": "npm:@wwa/socket.io-stream@^0.10.0",
@@ -124,7 +124,7 @@
         "@types/ejs": "^3.1.5",
         "@types/jest": "^30.0.0",
         "@types/js-yaml": "^4.0.9",
-        "@types/node": "^25.9.1",
+        "@types/node": "^26.0.0",
         "@types/proxy-from-env": "^1.0.4",
         "@types/semver": "^7.5.5",
         "@types/shelljs": "^0.10.0",
@@ -2865,6 +2865,30 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/@eslint/eslintrc/node_modules/js-yaml": {
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.2.0.tgz",
+      "integrity": "sha512-ePWsvanv0DWuDRsW8dnt+R4jQ31SCRCQ7hhNcPXZPsoBZiemuZNYGf7adZdqX2D86j6rvKp3RpCxVTSb8WQlOw==",
+      "dev": true,
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/puzrin"
+        },
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/nodeca"
+        }
+      ],
+      "license": "MIT",
+      "peer": true,
+      "dependencies": {
+        "argparse": "^2.0.1"
+      },
+      "bin": {
+        "js-yaml": "bin/js-yaml.js"
+      }
+    },
     "node_modules/@eslint/eslintrc/node_modules/minimatch": {
       "version": "3.1.5",
       "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz",
@@ -4570,12 +4594,12 @@
       "license": "MIT"
     },
     "node_modules/@types/node": {
-      "version": "25.9.3",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-25.9.3.tgz",
-      "integrity": "sha512-603BddQMv3pUcr4U2dhujk83N2tTDVr/34wII2B6bJy6g+8WD6yUb11jszNs0gdi4PesVWl7ABt8nYMVpnLUcg==",
+      "version": "26.0.0",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-26.0.0.tgz",
+      "integrity": "sha512-vf2YFi1iY9lHGwNJMs01biZFbKJkrZR1T6/MlzjhJLPdntOHLhTrDSnSVcdtvjihi4VQNlrFRIxLsDBlQpAipA==",
       "license": "MIT",
       "dependencies": {
-        "undici-types": ">=7.24.0 <7.24.7"
+        "undici-types": "~8.3.0"
       }
     },
     "node_modules/@types/proxy-from-env": {
@@ -11394,9 +11418,9 @@
       "license": "MIT"
     },
     "node_modules/js-yaml": {
-      "version": "4.2.0",
-      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.2.0.tgz",
-      "integrity": "sha512-ePWsvanv0DWuDRsW8dnt+R4jQ31SCRCQ7hhNcPXZPsoBZiemuZNYGf7adZdqX2D86j6rvKp3RpCxVTSb8WQlOw==",
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-5.0.0.tgz",
+      "integrity": "sha512-GSvaPUbk1U+FMZ7rJzF+F8e5YVtu7KnD40et/5rBXXRBv2jCO9L3qCewvIDDdudC0QycTFlf6EAA+h3kxBsuUw==",
       "funding": [
         {
           "type": "github",
@@ -11412,7 +11436,7 @@
         "argparse": "^2.0.1"
       },
       "bin": {
-        "js-yaml": "bin/js-yaml.js"
+        "js-yaml": "bin/js-yaml.mjs"
       }
     },
     "node_modules/jsdoc-type-pratt-parser": {
@@ -11668,6 +11692,28 @@
         "url": "https://github.com/sponsors/isaacs"
       }
     },
+    "node_modules/lando/node_modules/js-yaml": {
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.2.0.tgz",
+      "integrity": "sha512-ePWsvanv0DWuDRsW8dnt+R4jQ31SCRCQ7hhNcPXZPsoBZiemuZNYGf7adZdqX2D86j6rvKp3RpCxVTSb8WQlOw==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/puzrin"
+        },
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/nodeca"
+        }
+      ],
+      "license": "MIT",
+      "dependencies": {
+        "argparse": "^2.0.1"
+      },
+      "bin": {
+        "js-yaml": "bin/js-yaml.js"
+      }
+    },
     "node_modules/lando/node_modules/lru-cache": {
       "version": "11.2.7",
       "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.7.tgz",
@@ -13418,9 +13464,9 @@
       }
     },
     "node_modules/semver": {
-      "version": "7.8.4",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.4.tgz",
-      "integrity": "sha512-rUCObTnP32Q08R2uuIrt7r9PlEonuTmtuXYcW6s5kjdlj3xbnwe+21yXptAUYcMAABLkYYTtnmzb3w3EDZfueA==",
+      "version": "7.8.5",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz",
+      "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==",
       "license": "ISC",
       "bin": {
         "semver": "bin/semver.js"
@@ -14587,9 +14633,9 @@
       }
     },
     "node_modules/undici-types": {
-      "version": "7.24.6",
-      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz",
-      "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==",
+      "version": "8.3.0",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz",
+      "integrity": "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==",
       "license": "MIT"
     },
     "node_modules/unicode-canonical-property-names-ecmascript": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@automattic/vip",
-  "version": "4.0.5",
+  "version": "4.0.6",
   "description": "The VIP Javascript library & CLI",
   "main": "index.js",
   "bin": {
@@ -128,7 +128,7 @@
     "@types/ejs": "^3.1.5",
     "@types/jest": "^30.0.0",
     "@types/js-yaml": "^4.0.9",
-    "@types/node": "^25.9.1",
+    "@types/node": "^26.0.0",
     "@types/proxy-from-env": "^1.0.4",
     "@types/semver": "^7.5.5",
     "@types/shelljs": "^0.10.0",
@@ -159,13 +159,13 @@
     "graphql": "17.0.1",
     "graphql-tag": "2.12.7",
     "https-proxy-agent": "^9.0.0",
-    "js-yaml": "^4.1.0",
+    "js-yaml": "^5.0.0",
     "jwt-decode": "4.0.0",
     "lando": "github:automattic/lando-cli#78d382fcf40357c4a4cebe4a3cfaef032eab743a",
     "node-stream-zip": "1.15.0",
     "open": "^11.0.0",
     "proxy-from-env": "^2.0.0",
-    "semver": "7.8.4",
+    "semver": "7.8.5",
     "shelljs": "^0.10.0",
     "socket.io-client": "^4.5.3",
     "socket.io-stream": "npm:@wwa/socket.io-stream@^0.10.0",