gatsby-source-filesystem 2.1.46 → 2.1.48

package/CHANGELOG.md

@@ -3,6 +3,16 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [2.1.48](https://github.com/gatsbyjs/gatsby/compare/gatsby-source-filesystem@2.1.47...gatsby-source-filesystem@2.1.48) (2020-02-01)
+
+### Bug Fixes
+
+- **gatsby-source-filesystem:** Retry stalled remote file downloads ([#20843](https://github.com/gatsbyjs/gatsby/issues/20843)) ([536686b](https://github.com/gatsbyjs/gatsby/commit/536686b))
+
+## [2.1.47](https://github.com/gatsbyjs/gatsby/compare/gatsby-source-filesystem@2.1.46...gatsby-source-filesystem@2.1.47) (2020-01-29)
+
+**Note:** Version bump only for package gatsby-source-filesystem
+
 ## [2.1.46](https://github.com/gatsbyjs/gatsby/compare/gatsby-source-filesystem@2.1.45...gatsby-source-filesystem@2.1.46) (2020-01-09)
 
 **Note:** Version bump only for package gatsby-source-filesystem

package/create-remote-file-node.js

@@ -80,6 +80,10 @@ let totalJobs = 0;
 
 const CACHE_DIR = `.cache`;
 const FS_PLUGIN_DIR = `gatsby-source-filesystem`;
+const STALL_RETRY_LIMIT = 3;
+const STALL_TIMEOUT = 30000;
+const CONNECTION_RETRY_LIMIT = 5;
+const CONNECTION_TIMEOUT = 30000;
 /********************
  * Queue Management *
  ********************/
@@ -142,31 +146,64 @@ async function pushToQueue(task, cb) {
  * @param  {Headers}  headers
  * @param  {String}   tmpFilename
  * @param  {Object}   httpOpts
+ * @param  {number}   attempt
  * @return {Promise<Object>}  Resolves with the [http Result Object]{@link https://nodejs.org/api/http.html#http_class_http_serverresponse}
  */
 
 
-const requestRemoteNode = (url, headers, tmpFilename, httpOpts) => new Promise((resolve, reject) => {
-  const opts = Object.assign({}, {
-    timeout: 30000,
-    retries: 5
-  }, httpOpts);
+const requestRemoteNode = (url, headers, tmpFilename, httpOpts, attempt = 1) => new Promise((resolve, reject) => {
+  let timeout; // Called if we stall for 30s without receiving any data
+
+  const handleTimeout = async () => {
+    fsWriteStream.close();
+    fs.removeSync(tmpFilename);
+
+    if (attempt < STALL_RETRY_LIMIT) {
+      // Retry by calling ourself recursively
+      resolve(requestRemoteNode(url, headers, tmpFilename, httpOpts, attempt + 1));
+    } else {
+      reject(`Failed to download ${url} after ${STALL_RETRY_LIMIT} attempts`);
+    }
+  };
+
+  const resetTimeout = () => {
+    if (timeout) {
+      clearTimeout(timeout);
+    }
+
+    timeout = setTimeout(handleTimeout, STALL_TIMEOUT);
+  };
+
   const responseStream = got.stream(url, Object.assign({
-    headers
-  }, opts));
+    headers,
+    timeout: CONNECTION_TIMEOUT,
+    retries: CONNECTION_RETRY_LIMIT
+  }, httpOpts));
   const fsWriteStream = fs.createWriteStream(tmpFilename);
-  responseStream.pipe(fsWriteStream);
-  responseStream.on(`downloadProgress`, pro => console.log(pro)); // If there's a 400/500 response or other error.
+  responseStream.pipe(fsWriteStream); // If there's a 400/500 response or other error.
+
+  responseStream.on(`error`, error => {
+    if (timeout) {
+      clearTimeout(timeout);
+    }
 
-  responseStream.on(`error`, (error, body, response) => {
     fs.removeSync(tmpFilename);
     reject(error);
   });
   fsWriteStream.on(`error`, error => {
+    if (timeout) {
+      clearTimeout(timeout);
+    }
+
     reject(error);
   });
   responseStream.on(`response`, response => {
+    resetTimeout();
     fsWriteStream.on(`finish`, () => {
+      if (timeout) {
+        clearTimeout(timeout);
+      }
+
       resolve(response);
     });
   });

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "gatsby-source-filesystem",
   "description": "Gatsby plugin which parses files within a directory for further parsing by other plugins",
-  "version": "2.1.46",
+  "version": "2.1.48",
   "author": "Kyle Mathews <mathews.kyle@gmail.com>",
   "bugs": {
     "url": "https://github.com/gatsbyjs/gatsby/issues"
@@ -13,8 +13,8 @@
     "chokidar": "3.3.0",
     "file-type": "^12.4.0",
     "fs-extra": "^8.1.0",
-    "gatsby-core-utils": "^1.0.26",
-    "got": "^7.1.0",
+    "gatsby-core-utils": "^1.0.28",
+    "got": "^8.3.2",
     "md5-file": "^3.2.3",
     "mime": "^2.4.4",
     "pretty-bytes": "^5.3.0",
@@ -26,7 +26,7 @@
   "devDependencies": {
     "@babel/cli": "^7.7.5",
     "@babel/core": "^7.7.5",
-    "babel-preset-gatsby-package": "^0.2.15",
+    "babel-preset-gatsby-package": "^0.2.16",
     "cross-env": "^5.2.1"
   },
   "homepage": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-filesystem#readme",
@@ -52,5 +52,5 @@
   "engines": {
     "node": ">=8.0.0"
   },
-  "gitHead": "4b6e778f75f531dba4f7dbe6c0af7daec1deac2a"
+  "gitHead": "b349d1096e65f2f3738e7230184d288264b03341"
 }