fetch-har 5.0.5 → 6.0.0

package/README.md

@@ -13,13 +13,17 @@ npm install --save fetch-har
 
 ## Usage
 ```js
-const fetchHar = require('fetch-har');
+require('isomorphic-fetch');
+const fetchHar = require('.');
 
-// If executing from an environment without `fetch`, you'll need to polyfill.
-global.fetch = require('node-fetch');
-global.Headers = require('node-fetch').Headers;
-global.Request = require('node-fetch').Request;
-global.FormData = require('form-data');
+// If executing from an environment that dodoesn't normally provide fetch()
+// you'll need to polyfill some APIs in order to make `multipart/form-data`
+// requests.
+if (!globalThis.FormData) {
+  globalThis.Blob = require('formdata-node').Blob;
+  globalThis.File = require('formdata-node').File;
+  globalThis.FormData = require('formdata-node').FormData;
+}
 
 const har = {
   log: {
@@ -36,7 +40,10 @@ const har = {
               value: 'application/json',
             },
           ],
-          queryString: [{ name: 'a', value: 1 }, { name: 'b', value: 2 }],
+          queryString: [
+            { name: 'a', value: 1 },
+            { name: 'b', value: 2 },
+          ],
           postData: {
             mimeType: 'application/json',
             text: '{"id":8,"category":{"id":6,"name":"name"},"name":"name"}',
@@ -50,22 +57,48 @@ const har = {
 };
 
 fetchHar(har)
-  .then(request => request.json())
+  .then(res => res.json())
   .then(console.log);
 ```
 
-### `fetchHar(har, userAgent) => Promise`
+### API
+If you are executing `fetch-har` in a browser environment that supports the [FormData API](https://developer.mozilla.org/en-US/docs/Web/API/FormData) then you don't need to do anything. If you arent, however, you'll need to polyfill it.
+
+Unfortunately the most popular NPM package [form-data](https://npm.im/form-data) ships with a [non-spec compliant API](https://github.com/form-data/form-data/issues/124), and for this we don't recommend you use it, as if you use `fetch-har` to upload files it may not work.
+
+We recommend either [formdata-node](https://npm.im/formdata-node) or [formdata-polyfill](https://npm.im/formdata-polyfill).
+
+#### Options
+##### userAgent
+A custom `User-Agent` header to apply to your request. Please note that browsers have their own handling for these headers in `fetch()` calls so it may not work everywhere; it will always be sent in Node however.
+
+```js
+await fetchHar(har, { userAgent: 'my-client/1.0' });
+```
+
+##### files
+An optional object map you can supply to use for `multipart/form-data` file uploads in leu of relying on if the HAR you have has [data URLs](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs). It supports Node file buffers and the [File](https://developer.mozilla.org/en-US/docs/Web/API/File) API.
+
+```js
+await fetchHar(har, { files: {
+  'owlbert.png': await fs.readFile('./owlbert.png'),
+  'file.txt': document.querySelector('#some-file-input').files[0],
+} });
+```
 
-- `har` is a [har](https://en.wikipedia.org/wiki/.har) file format.
-- `userAgent` is an optional user agent string to let you declare where the request is coming from.
+If you don't supply this option `fetch-har` will fallback to the data URL present within the supplied HAR. If no `files` option is present, and no data URL (via `param.value`) is present in the HAR, a fatal exception will be thrown.
 
-Performs a fetch request from a given HAR definition. HAR definitions can be used to list lots of requests but we only use the first from the `log.entries` array.
+##### multipartEncoder
+> ❗ If you are using `fetch-har` in Node you may need this option!
 
-### `fetchHar.constructRequest(har, userAgent) => Request`
+If you are running `fetch-har` within a Node environment and you're using `node-fetch@2`, or another `fetch` polyfill that does not support a spec-compliant `FormData` API, you will need to specify an encoder that will transform your `FormData` object into something that can be used with [Request.body](https://developer.mozilla.org/en-US/docs/Web/API/Request/body).
 
-- `har` is a [har](https://en.wikipedia.org/wiki/.har) file format.
-- `userAgent` is an optional user agent string to let you declare where the request is coming from.
+We recommend [form-data-encoder](https://npm.im/form-data-encoder).
 
-We also export a second function which is used to construct a [Request](https://developer.mozilla.org/en-US/docs/Web/API/Request) object from your HAR.
+```js
+const { FormDataEncoder } = require('form-data-encoder');
+
+await fetchHar(har, { multipartEncoder: FormDataEncoder });
+```
 
-This function is mainly exported for testing purposes but could be useful if you want to construct a request but do not want to execute it right away.
+You do **not**, and shouldn't, need to use this option in browser environments.

package/example.js

@@ -1,11 +1,14 @@
 /* eslint-disable import/no-extraneous-dependencies, no-console */
+require('isomorphic-fetch');
 const fetchHar = require('.');
 
-// If executing from an environment without `fetch`, you'll need to polyfill
-global.fetch = require('node-fetch');
-global.Headers = require('node-fetch').Headers;
-global.Request = require('node-fetch').Request;
-global.FormData = require('form-data');
+// If executing from an environment that dodoesn't normally provide fetch() you'll need to polyfill some APIs in order
+// to make `multipart/form-data` requests.
+if (!globalThis.FormData) {
+  globalThis.Blob = require('formdata-node').Blob;
+  globalThis.File = require('formdata-node').File;
+  globalThis.FormData = require('formdata-node').FormData;
+}
 
 const har = {
   log: {
@@ -39,5 +42,5 @@ const har = {
 };
 
 fetchHar(har)
-  .then(request => request.json())
+  .then(res => res.json())
   .then(console.log);

package/index.js

@@ -1,7 +1,76 @@
-/* eslint-disable no-case-declarations */
+const { Readable } = require('readable-stream');
 const parseDataUrl = require('parse-data-url');
 
-function constructRequest(har, userAgent = false) {
+if (!globalThis.Blob) {
+  try {
+    // eslint-disable-next-line import/no-extraneous-dependencies
+    globalThis.Blob = require('formdata-node').Blob;
+  } catch (e) {
+    throw new Error(
+      'Since you do not have the Blob API available in this environment you must install the optional `formdata-node` dependency.'
+    );
+  }
+}
+
+if (!globalThis.File) {
+  try {
+    // eslint-disable-next-line import/no-extraneous-dependencies
+    globalThis.File = require('formdata-node').File;
+  } catch (e) {
+    throw new Error(
+      'Since you do not have the File API available in this environment you must install the optional `formdata-node` dependency.'
+    );
+  }
+}
+
+function isBrowser() {
+  return typeof window !== 'undefined' && typeof document !== 'undefined';
+}
+
+function isBuffer(value) {
+  return typeof Buffer !== 'undefined' && Buffer.isBuffer(value);
+}
+
+function isFile(value) {
+  if (value instanceof File) {
+    // The `Blob` polyfill on Node comes back as being an instanceof `File`. Because passing a Blob into
+    // a File will end up with a corrupted file we want to prevent this.
+    //
+    // This object identity crisis does not happen in the browser.
+    return value.constructor.name === 'File';
+  }
+
+  return false;
+}
+
+/**
+ * @license MIT
+ * @see {@link https://github.com/octet-stream/form-data-encoder/blob/master/lib/util/isFunction.ts}
+ */
+function isFunction(value) {
+  return typeof value === 'function';
+}
+
+/**
+ * We're loading this library in here instead of loading it from `form-data-encoder` because that uses lookbehind
+ * regex in its main encoder that Safari doesn't support so it throws a fatal page exception.
+ *
+ * @license MIT
+ * @see {@link https://github.com/octet-stream/form-data-encoder/blob/master/lib/util/isFormData.ts}
+ */
+function isFormData(value) {
+  return (
+    value &&
+    isFunction(value.constructor) &&
+    value[Symbol.toStringTag] === 'FormData' && // eslint-disable-line compat/compat
+    isFunction(value.append) &&
+    isFunction(value.getAll) &&
+    isFunction(value.entries) &&
+    isFunction(value[Symbol.iterator]) // eslint-disable-line compat/compat
+  );
+}
+
+function constructRequest(har, opts = { userAgent: false, files: false, multipartEncoder: false }) {
   if (!har) throw new Error('Missing HAR definition');
   if (!har.log || !har.log.entries || !har.log.entries.length) throw new Error('Missing log.entries array');
 
@@ -31,7 +100,7 @@ function constructRequest(har, userAgent = false) {
     // As the browser fetch API can't set custom cookies for requests, they instead need to be defined on the document
     // and passed into the request via `credentials: include`. Since this is a browser-specific quirk, that should only
     // happen in browsers!
-    if (typeof window !== 'undefined' && typeof document !== 'undefined') {
+    if (isBrowser()) {
       request.cookies.forEach(cookie => {
         document.cookie = `${encodeURIComponent(cookie.name)}=${encodeURIComponent(cookie.value)}`;
       });
@@ -68,7 +137,10 @@ function constructRequest(har, userAgent = false) {
           options.body = encodedParams.toString();
           break;
 
+        case 'multipart/alternative':
         case 'multipart/form-data':
+        case 'multipart/mixed':
+        case 'multipart/related':
           // If there's a Content-Type header set remove it. We're doing this because when we pass the form data object
           // into `fetch` that'll set a proper `Content-Type` header for this request that also includes the boundary
           // used on the content.
@@ -79,59 +151,93 @@ function constructRequest(har, userAgent = false) {
             headers.delete('Content-Type');
           }
 
-          // The `form-data` NPM module returns one of two things: a native `FormData` API, or its own polyfill. Since
-          // the polyfill does not support the full API of the native FormData object, when you load `form-data` within
-          // a browser environment you'll have two major differences in API:
-          //
-          //  * The `.append()` API in `form-data` requires that the third argument is an object containing various,
-          //    undocumented, options. In the browser, `.append()`'s third argument should only be present when the
-          //    second is a `Blob` or `USVString`, and when it is present, it should be a filename string.
-          //  * `form-data` does not expose an `.entries()` API, so the only way to retrieve data out of it for
-          //    construction of boundary-separated payload content is to use its `.pipe()` API. Since the browser
-          //    doesn't have this API, you'll be unable to retrieve data out of it.
-          //
-          // Now since the native `FormData` API is iterable, and has the `.entries()` iterator, we can easily detect
-          // what version of the FormData API we have access to by looking for this and constructing a simple wrapper
-          // to disconnect some of this logic so you can work against a single, consistent API.
-          //
-          // Having to do this isn't fun, but it's the only way you can write code to work with `multipart/form-data`
-          // content under a server and browser.
           const form = new FormData();
-          const isNativeFormData = typeof form[Symbol.iterator] === 'function';
+          if (!isFormData(form)) {
+            // The `form-data` NPM module returns one of two things: a native `FormData` API or its own polyfill.
+            // Unfortunately this polyfill does not support the full API of the native FormData object so when you load
+            // `form-data` within a browser environment you'll have two major differences in API:
+            //
+            //  * The `.append()` API in `form-data` requires that the third argument is an object containing various,
+            //    undocumented, options. In the browser, `.append()`'s third argument should only be present when the
+            //    second is a `Blob` or `USVString`, and when it is present, it should be a filename string.
+            //  * `form-data` does not expose an `.entries()` API, so the only way to retrieve data out of it for
+            //    construction of boundary-separated payload content is to use its `.pipe()` API. Since the browser
+            //    doesn't have this API, you'll be unable to retrieve data out of it.
+            //
+            // Now since the native `FormData` API is iterable, and has the `.entries()` iterator, we can easily detect
+            // if we have a native copy of the FormData API. It's for all of these reasons that we're opting to hard
+            // crash here because supporting this non-compliant API is more trouble than its worth.
+            //
+            // https://github.com/form-data/form-data/issues/124
+            throw new Error(
+              "We've detected you're using a non-spec compliant FormData library. We recommend polyfilling FormData with https://npm.im/formdata-node"
+            );
+          }
 
           request.postData.params.forEach(param => {
-            if ('fileName' in param && !('value' in param)) {
-              throw new Error(
-                "The supplied HAR has a postData parameter with `fileName`, but no `value` content. Since this library doesn't have access to the filesystem, it can't fetch that file."
-              );
-            }
-
-            // If the incoming parameter is a file, and that files value is a data URL, we should decode that and set
-            // the contents of the value in the HAR to the actual contents of the file.
             if ('fileName' in param) {
-              const parsed = parseDataUrl(param.value);
-              if (parsed) {
-                // eslint-disable-next-line no-param-reassign
-                param.value = parsed.toBuffer().toString();
-              }
-            }
+              if (opts.files && param.fileName in opts.files) {
+                const fileContents = opts.files[param.fileName];
+
+                // If the file we've got available to us is a Buffer then we need to convert it so that the FormData
+                // API can use it.
+                if (isBuffer(fileContents)) {
+                  form.set(
+                    param.name,
+                    new File([fileContents], param.fileName, {
+                      type: param.contentType || null,
+                    }),
+                    param.fileName
+                  );
+
+                  return;
+                } else if (isFile(fileContents)) {
+                  form.set(param.name, fileContents, param.fileName);
+                  return;
+                }
+
+                throw new TypeError(
+                  'An unknown object has been supplied into the `files` config for use. We only support instances of the File API and Node Buffer objects.'
+                );
+              } else if ('value' in param) {
+                let paramBlob;
+                const parsed = parseDataUrl(param.value);
+                if (parsed) {
+                  // If we were able to parse out this data URL we don't need to transform its data into a buffer for
+                  // `Blob` because that supports data URLs already.
+                  paramBlob = new Blob([param.value], { type: parsed.contentType || param.contentType || null });
+                } else {
+                  paramBlob = new Blob([param.value], { type: param.contentType || null });
+                }
 
-            if (isNativeFormData) {
-              if ('fileName' in param) {
-                const paramBlob = new Blob([param.value], { type: param.contentType || null });
                 form.append(param.name, paramBlob, param.fileName);
-              } else {
-                form.append(param.name, param.value);
+                return;
               }
-            } else {
-              form.append(param.name, param.value || '', {
-                filename: param.fileName || null,
-                contentType: param.contentType || null,
-              });
+
+              throw new Error(
+                "The supplied HAR has a postData parameter with `fileName`, but neither `value` content within the HAR or any file buffers were supplied with the `files` option. Since this library doesn't have access to the filesystem, it can't fetch that file."
+              );
             }
+
+            form.append(param.name, param.value);
           });
 
-          options.body = form;
+          // If a the `fetch` polyfill that's being used here doesn't have spec-compliant handling for the `FormData`
+          // API (like `node-fetch@2`), then you should pass in a handler (like the `form-data-encoder` library) to
+          // transform its contents into something that can be used with the `Request` object.
+          //
+          // https://www.npmjs.com/package/formdata-node
+          if (opts.multipartEncoder) {
+            // eslint-disable-next-line new-cap
+            const encoder = new opts.multipartEncoder(form);
+            Object.keys(encoder.headers).forEach(header => {
+              headers.set(header, encoder.headers[header]);
+            });
+
+            options.body = Readable.from(encoder);
+          } else {
+            options.body = form;
+          }
           break;
 
         default:
@@ -148,27 +254,59 @@ function constructRequest(har, userAgent = false) {
 
           options.body = JSON.stringify(formBody);
       }
-    } else {
-      options.body = request.postData.text;
+    } else if (request.postData.text.length) {
+      // If we've got `files` map content present, and this post data content contains a valid data URL then we can
+      // substitute the payload with that file instead of the using data URL.
+      if (opts.files) {
+        const parsed = parseDataUrl(request.postData.text);
+        if (parsed && 'name' in parsed && parsed.name in opts.files) {
+          const fileContents = opts.files[parsed.name];
+          if (isBuffer(fileContents)) {
+            options.body = fileContents;
+          } else if (isFile(fileContents)) {
+            // `Readable.from` isn't available in browsers but the browser `Request` object can handle `File` objects
+            // just fine without us having to mold it into shape.
+            if (isBrowser()) {
+              options.body = fileContents;
+            } else {
+              options.body = Readable.from(fileContents.stream());
+
+              // Supplying a polyfilled `File` stream into `Request.body` doesn't automatically add `Content-Length`.
+              if (!headers.has('content-length')) {
+                headers.set('content-length', fileContents.size);
+              }
+            }
+          }
+        }
+      }
+
+      if (typeof options.body === 'undefined') {
+        options.body = request.postData.text;
+      }
     }
   }
 
   if ('queryString' in request && request.queryString.length) {
-    const query = request.queryString.map(q => `${q.name}=${q.value}`).join('&');
-    querystring = `?${query}`;
+    const queryParams = new URL(url).searchParams;
+
+    request.queryString.forEach(q => {
+      queryParams.append(q.name, q.value);
+    });
+
+    querystring = queryParams.toString();
   }
 
-  if (userAgent) {
-    headers.append('User-Agent', userAgent);
+  if (opts.userAgent) {
+    headers.append('User-Agent', opts.userAgent);
   }
 
   options.headers = headers;
 
-  return new Request(`${url}${querystring}`, options);
+  return new Request(`${url.split('?')[0]}${querystring ? `?${querystring}` : ''}`, options);
 }
 
-function fetchHar(har, userAgent) {
-  return fetch(constructRequest(har, userAgent));
+function fetchHar(har, opts = { userAgent: false, files: false, multipartEncoder: false }) {
+  return fetch(constructRequest(har, opts));
 }
 
 module.exports = fetchHar;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "fetch-har",
-  "version": "5.0.5",
+  "version": "6.0.0",
   "description": "Make a fetch request from a HAR definition",
   "main": "index.js",
   "engines": {
@@ -12,7 +12,10 @@
     "prettier": "prettier --list-different --write \"./**.js\"",
     "release": "npx conventional-changelog-cli -i CHANGELOG.md -s",
     "serve": "node __tests__/server.js",
-    "test": "jest --coverage"
+    "test:browser": "karma start --single-run",
+    "test:browser:chrome": "karma start --browsers=Chrome --single-run=false",
+    "test:browser:debug": "karma start --single-run=false",
+    "test": "nyc mocha"
   },
   "repository": {
     "type": "git",
@@ -24,37 +27,32 @@
   },
   "homepage": "https://github.com/readmeio/fetch-har#readme",
   "dependencies": {
-    "parse-data-url": "^4.0.1"
+    "parse-data-url": "^4.0.1",
+    "readable-stream": "^3.6.0"
+  },
+  "optionalDependencies": {
+    "fromdata-node": "^4.3.2"
   },
   "devDependencies": {
-    "@readme/eslint-config": "^8.0.2",
-    "body-parser": "^1.19.0",
-    "cookie-parser": "^1.4.5",
-    "eslint": "^8.2.0",
-    "eslint-plugin-compat": "^4.0.0",
-    "express": "^4.17.1",
+    "@jsdevtools/host-environment": "^2.1.2",
+    "@jsdevtools/karma-config": "^3.1.7",
+    "@readme/eslint-config": "^8.1.2",
+    "chai": "^4.3.4",
+    "eslint": "^8.7.0",
+    "eslint-plugin-compat": "^4.0.1",
+    "eslint-plugin-mocha": "^10.0.3",
     "form-data": "^4.0.0",
-    "har-examples": "^2.0.1",
-    "jest": "^27.2.0",
-    "jest-puppeteer": "^6.0.0",
-    "multer": "^1.4.2",
-    "nock": "^13.1.1",
+    "form-data-encoder": "^1.7.1",
+    "formdata-node": "^4.3.2",
+    "har-examples": "^3.0.0",
+    "isomorphic-fetch": "^3.0.0",
+    "mocha": "^9.1.4",
     "node-fetch": "^2.6.0",
-    "prettier": "^2.4.1",
-    "webpack": "^5.53.0",
-    "webpack-dev-middleware": "^5.1.0"
+    "nyc": "^15.1.0",
+    "prettier": "^2.5.1"
   },
   "browserslist": [
     "last 2 versions"
   ],
-  "prettier": "@readme/eslint-config/prettier",
-  "jest": {
-    "preset": "jest-puppeteer",
-    "globals": {
-      "SERVER_URL": "http://localhost:4444"
-    },
-    "testMatch": [
-      "<rootDir>/__tests__/*.test.js"
-    ]
-  }
+  "prettier": "@readme/eslint-config/prettier"
 }

package/jest-puppeteer.config.js

@@ -1,6 +0,0 @@
-module.exports = {
-  server: {
-    command: 'npm run serve',
-    port: 4444,
-  },
-};