gatsby-source-filesystem 2.0.6 → 2.0.10

package/CHANGELOG.md

@@ -3,6 +3,32 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+<a name="2.0.10"></a>
+
+## [2.0.10](https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-filesystem/compare/gatsby-source-filesystem@2.0.9...gatsby-source-filesystem@2.0.10) (2018-11-29)
+
+**Note:** Version bump only for package gatsby-source-filesystem
+
+<a name="2.0.9"></a>
+
+## [2.0.9](https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-filesystem/compare/gatsby-source-filesystem@2.0.8...gatsby-source-filesystem@2.0.9) (2018-11-26)
+
+### Bug Fixes
+
+- **gatsby-plugin-filesystem:** throw meaningful errors on bad inputs ([#10123](https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-filesystem/issues/10123)) ([21ebf2c](https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-filesystem/commit/21ebf2c)), closes [#6643](https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-filesystem/issues/6643)
+
+<a name="2.0.8"></a>
+
+## [2.0.8](https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-filesystem/compare/gatsby-source-filesystem@2.0.7...gatsby-source-filesystem@2.0.8) (2018-11-08)
+
+**Note:** Version bump only for package gatsby-source-filesystem
+
+<a name="2.0.7"></a>
+
+## [2.0.7](https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-filesystem/compare/gatsby-source-filesystem@2.0.6...gatsby-source-filesystem@2.0.7) (2018-11-01)
+
+**Note:** Version bump only for package gatsby-source-filesystem
+
 <a name="2.0.6"></a>
 
 ## [2.0.6](https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-filesystem/compare/gatsby-source-filesystem@2.0.5...gatsby-source-filesystem@2.0.6) (2018-10-29)

package/README.md

@@ -175,6 +175,10 @@ createRemoteFileNode({
   // OPTIONAL
   // Adds htaccess authentication to the download request if passed in.
   auth: { htaccess_user: `USER`, htaccess_pass: `PASSWORD` },
+
+  // OPTIONAL
+  // Sets the file extension
+  ext: ".jpg",
 })
 ```
 
@@ -222,3 +226,20 @@ exports.downloadMediaFiles = ({
 ```
 
 The file node can then be queried using GraphQL. See an example of this in the [gatsby-source-wordpress README](/packages/gatsby-source-wordpress/#image-processing) where downloaded images are queried using [gatsby-transformer-sharp](/packages/gatsby-transformer-sharp/) to use in the component [gatsby-image](/packages/gatsby-image/).
+
+#### Retrieving the remote file extension
+
+The helper tries first to retrieve the file extension by parsing the url and the path provided (e.g. if the url is https://example.com/image.jpg, the extension will be inferred as `.jpg`). If the url does not contain an extension, we use the [`file-type`](https://www.npmjs.com/package/file-type) package to infer the file type. Finally, the extension _can_ be explicitly passed, like so:
+
+```javascript
+createRemoteFileNode({
+  // The source url of the remote file
+  url: `https://example.com/a-file-without-an-extension`,
+  store,
+  cache,
+  createNode,
+  createNodeId,
+  // if necessary!
+  ext: ".jpg",
+})
+```

package/create-remote-file-node.js

@@ -17,6 +17,10 @@ const _require = require(`valid-url`),
 
 const Queue = require(`better-queue`);
 
+const readChunk = require(`read-chunk`);
+
+const fileType = require(`file-type`);
+
 const _require2 = require(`./create-file-node`),
       createFileNode = _require2.createFileNode;
 
@@ -135,7 +139,6 @@ function pushToQueue(_x, _x2) {
  * @param  {String}   url
  * @param  {Headers}  headers
  * @param  {String}   tmpFilename
- * @param  {String}   filename
  * @return {Promise<Object>}  Resolves with the [http Result Object]{@link https://nodejs.org/api/http.html#http_class_http_serverresponse}
  */
 
@@ -152,7 +155,7 @@ function _pushToQueue() {
   return _pushToQueue.apply(this, arguments);
 }
 
-const requestRemoteNode = (url, headers, tmpFilename, filename) => new Promise((resolve, reject) => {
+const requestRemoteNode = (url, headers, tmpFilename) => new Promise((resolve, reject) => {
   const responseStream = got.stream(url, Object.assign({}, headers, {
     timeout: 30000,
     retries: 5
@@ -199,7 +202,8 @@ function _processRemoteNode() {
     cache,
     createNode,
     auth = {},
-    createNodeId
+    createNodeId,
+    ext
   }) {
     // Ensure our cache directory exists.
     const programDir = store.getState().program.directory;
@@ -220,14 +224,28 @@ function _processRemoteNode() {
 
 
     const digest = createHash(url);
-    const ext = getRemoteFileExtension(url);
-    const tmpFilename = createFilePath(programDir, `tmp-${digest}`, ext);
-    const filename = createFilePath(programDir, digest, ext); // Fetch the file.
+
+    if (!ext) {
+      ext = getRemoteFileExtension(url);
+    }
+
+    const tmpFilename = createFilePath(programDir, `tmp-${digest}`, ext); // Fetch the file.
 
     try {
-      const response = yield requestRemoteNode(url, headers, tmpFilename, filename); // Save the response headers for future requests.
+      const response = yield requestRemoteNode(url, headers, tmpFilename); // Save the response headers for future requests.
+
+      yield cache.set(cacheId(url), response.headers); // If the user did not provide an extension and we couldn't get one from remote file, try and guess one
+
+      if (ext === ``) {
+        const buffer = readChunk.sync(tmpFilename, 0, fileType.minimumBytes);
+        const filetype = fileType(buffer);
+
+        if (filetype) {
+          ext = `.${filetype.ext}`;
+        }
+      }
 
-      cache.set(cacheId(url), response.headers); // If the status code is 200, move the piped temp file to the real name.
+      const filename = createFilePath(programDir, digest, ext); // If the status code is 200, move the piped temp file to the real name.
 
       if (response.statusCode === 200) {
         yield fs.move(tmpFilename, filename, {
@@ -297,10 +315,30 @@ module.exports = ({
   cache,
   createNode,
   auth = {},
-  createNodeId
+  createNodeId,
+  ext = null
 }) => {
-  // Check if we already requested node for this remote file
+  // validation of the input
+  // without this it's notoriously easy to pass in the wrong `createNodeId`
+  // see gatsbyjs/gatsby#6643
+  if (typeof createNodeId !== `function`) {
+    throw new Error(`createNodeId must be a function, was ${typeof createNodeId}`);
+  }
+
+  if (typeof createNode !== `function`) {
+    throw new Error(`createNode must be a function, was ${typeof createNode}`);
+  }
+
+  if (typeof store !== `object`) {
+    throw new Error(`store must be the redux store, was ${typeof store}`);
+  }
+
+  if (typeof cache !== `object`) {
+    throw new Error(`cache must be the Gatsby cache, was ${typeof cache}`);
+  } // Check if we already requested node for this remote file
   // and return stored promise if we did.
+
+
   if (processingCache[url]) {
     return processingCache[url];
   }
@@ -317,6 +355,7 @@ module.exports = ({
     cache,
     createNode,
     createNodeId,
-    auth
+    auth,
+    ext
   });
 };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "gatsby-source-filesystem",
   "description": "Gatsby plugin which parses files within a directory for further parsing by other plugins",
-  "version": "2.0.6",
+  "version": "2.0.10",
   "author": "Kyle Mathews <mathews.kyle@gmail.com>",
   "bugs": {
     "url": "https://github.com/gatsbyjs/gatsby/issues"
@@ -11,11 +11,13 @@
     "better-queue": "^3.8.7",
     "bluebird": "^3.5.0",
     "chokidar": "^1.7.0",
+    "file-type": "^10.2.0",
     "fs-extra": "^5.0.0",
     "got": "^7.1.0",
     "md5-file": "^3.1.1",
     "mime": "^2.2.0",
     "pretty-bytes": "^4.0.2",
+    "read-chunk": "^3.0.0",
     "slash": "^1.0.0",
     "valid-url": "^1.0.9",
     "xstate": "^3.1.0"
@@ -23,7 +25,7 @@
   "devDependencies": {
     "@babel/cli": "^7.0.0",
     "@babel/core": "^7.0.0",
-    "babel-preset-gatsby-package": "^0.1.2",
+    "babel-preset-gatsby-package": "^0.1.3",
     "cross-env": "^5.1.4"
   },
   "homepage": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-filesystem#readme",
@@ -41,5 +43,5 @@
     "prepare": "cross-env NODE_ENV=production npm run build",
     "watch": "babel -w src --out-dir . --ignore **/__tests__"
   },
-  "gitHead": "dbd71c33ced0829c5f4bc552127ef80b4a7398a5"
+  "gitHead": "a3e548956009fecbcbd5e55f11e7803cbf5253d0"
 }