http-snapshotter 0.1.0

package/README.md

@@ -0,0 +1,170 @@
+# HTTP Snapshotter
+
+Take snapshots of HTTP requests for purpose of tests.
+
+WARNING: This module isn't concurrent or thread safe yet. You can only use it on serial test runners like `tape`.
+
+Example (test.js):
+
+```js
+import test from "tape";
+import { fileURLToPath } from "node:url";
+import { resolve, dirname } from "node:path";
+import { start } from "http-snapshotter";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const snapshotDirectory = resolve(__dirname, "http-snapshots");
+
+await start({ snapshotDirectory });
+
+test("Latest XKCD comic (ESM)", async (t) => {
+  const res = await fetch("https://xkcd.com/info.0.json");
+  const json = await res.json();
+
+  t.deepEquals(json.title, "Iceberg Efficiency", "must be equal");
+});
+```
+
+
+To create snapshots the first time run:
+```sh
+SNAPSHOT=update node test.js
+```
+
+You will see a file named `get-xkcd-com-info-0-arAlFb5gfcr9aCN.json` created in the `http-snapshots` directory.
+
+e.g.
+
+Then onwards running: `node test.js` or `SNAPSHOT=read node test.js` will ensure HTTP network calls are all read from a snapshot file.
+It will prevent any real HTTP calls from happening by failing the test (if it didn't have a snapshot file).
+
+## About snapshot files and its name
+
+A snapshot file name unique identifies a request. By default it is a combination of HTTP method + URL + body that makes a request unique.
+The hash of concatenated HTTP method + URL + body makes the file name suffix.
+
+However you may want to specially handle some requests. e.g. DynamoDB calls also need the `x-amz-target` header to uniquely identify the request,
+because every call is a POST call with DynamoDB. You can add logic to create better snapshot files for this case:
+
+```js
+import {
+  start,
+  defaultSnapshotFileNameGenerator,
+  attachSnapshotFilenameGenerator
+} from "http-snapshotter";
+const slugify = require('@sindresorhus/slugify');
+
+/**
+ * @param {Request} request https://developer.mozilla.org/en-US/docs/Web/API/Request
+ */
+async function mySnapshotFilenameGenerator(request) {
+  const url = new URL(request.url);
+  if (!url.hostname.startsWith('dynamodb.') || !url.hostname.endsWith('.amazonaws.com')) {
+    return defaultSnapshotFileNameGenerator(request);
+  }
+
+  // Make a more readable file name suffix
+  let filePrefix;
+  const xAmzHeader = request.headers?.get?.('x-amz-target')?.split?.('.')?.pop?.() || '';
+  filePrefix = [
+    'dynamodb',
+    slugify(xAmzHeader),
+    slugify(JSON.parse(await request.clone().text())?.TableName),
+  ].filter(Boolean).join('-');
+
+  // Input data
+  const dataList = await Promise.all(
+    ['url', 'body'].map((key) => {
+      if (key === 'body') {
+        return request.clone().text();
+      }
+      return request[key];
+    }),
+  );
+
+  return {
+    filePrefix,
+    fileSuffixKey: `${xAmzHeader}#${dataList.join('#')}`,
+  };
+}
+
+attachSnapshotFilenameGenerator(mySnapshotFilenameGenerator);
+```
+
+## Same request, varied response
+
+There are scenarios where one needs to test varied response for the same call (.e.g GET /account).
+
+There are 2 ways to go about this.
+
+Method 1: The easy way it to not touch the existing snapshot file, and use `attachResponseTransformer` to
+change the response on runtime for the specific test:
+
+```js
+import {
+  // ...
+  attachResponseTransformer,
+  resetResponseTransformer,
+} from "http-snapshotter";
+
+test('Test behavior on a free account', async (t) => {
+  /**
+   * @param {Response} response https://developer.mozilla.org/en-US/docs/Web/API/Response
+   * @param {Request} request https://developer.mozilla.org/en-US/docs/Web/API/Request
+   */
+  const interceptResponse = async (response, request) => {
+    const url = new URL(request.url);
+    if (request.method === 'GET' && url.pathname === '/account') {
+      return new Response(
+        JSON.stringify({
+          ...(await response.clone().json()),
+          free_user: true,
+        }),
+        {
+          headers: response.headers
+        }
+      )
+    }
+    // 
+    return response;
+  };
+  attachResponseTransformer(interceptResponse);
+
+  // make fetch() call here
+  // assert the test
+
+  // cleanup before moving to next test
+  resetResponseTransformer();
+});
+```
+
+Method 2: Add a filename suffix for the specific test you are running and manually edit the new snapshot file (it is a regular JSON file)
+
+(building on the last attachSnapshotFilenameGenerator snippet)
+
+```js
+test('Test behavior on a free account', async (t) => {
+  attachSnapshotFilenameGenerator(async (request) => {
+    const defaultReturn = mySnapshotFilenameGenerator();
+
+    const url = new URL(request.url);
+    if (request.method === 'GET' && url.pathname === '/account') {
+      return {
+        filePrefix: `free-account-test-${defaultReturn.filePrefix}`,
+        fileSuffixKey: defaultReturn.fileSuffixKey,
+      };
+    }
+
+    return defaultReturn;
+  });
+
+  // make fetch() call here
+  // assert the test
+
+  // cleanup before moving to next test
+  attachSnapshotFilenameGenerator(mySnapshotFilenameGenerator);
+});
+```
+
+Now when you run `SNAPHOT=update node specific-test.js` you will get a snapshot file with `free-account-test-` as prefix. You can now edit the JSON response for this test.

package/index.js

@@ -0,0 +1,371 @@
+/* eslint-disable import/no-extraneous-dependencies, no-console */
+/**
+ * How to use:
+ * Place file in directory as tests and add `require('./snapshotter').start()` before tests begin
+ *
+ * WARNING: This snapshotter is not thread-safe. Only will work with test runners like tape where
+ * tests run on single threads.
+ *
+ * Run tests with environment variable SNAPSHOT=update first time to create snapshots
+ * SNAPSHOT=update <test runner command>
+ * e.g. SNAPSHOT=update tape tests/**\/*.js | tap-diff
+ *
+ * Here onwards run test runner without SNAPSHOT env variable or SNAPSHOT=read
+ * You can use SNAPSHOT=ignore to neither read not write snapshots, for testing on real
+ * network operations.
+ *
+ * Unused snapshot files will be written into a log file named 'unused-snapshots.log'.
+ * You can delete those files manually.
+ *
+ * Log requests with LOG_REQ=1 env variable
+ * or node.js built-in NODE_DEBUG=http,http2
+ *
+ * More docs at the end of this file, find the exported methods.
+ */
+// Tested with @mswjs/interceptors v0.24.1
+const { BatchInterceptor } = require('@mswjs/interceptors');
+const { ClientRequestInterceptor } = require('@mswjs/interceptors/ClientRequest');
+const { FetchInterceptor } = require('@mswjs/interceptors/fetch');
+const slugify = require('@sindresorhus/slugify');
+const { createHash } = require('node:crypto');
+const { promises: fs } = require('node:fs');
+const { resolve } = require('node:path');
+
+// Environment variable SNAPSHOT = update / ignore / read (default)
+const SNAPSHOT = process.env.SNAPSHOT || 'read';
+const LOG_REQ = process.env.LOG_REQ === '1' || process.env.LOG_REQ === 'true';
+const unusedSnapshotsLogFile = 'unused-snapshots.log';
+let snapshotDirectory = null;
+
+/**
+ * @typedef SnapshotText
+ * @property {'text'} responseType
+ * @property {string} fileSuffixKey
+ * @property {object} request
+ * @property {string} request.method
+ * @property {string} request.url
+ * @property {string[][]} request.headers
+ * @property {string|undefined} request.body
+ * @property {object} response
+ * @property {number} response.status
+ * @property {string} response.statusText
+ * @property {string[][]} response.headers
+ * @property {string|undefined} response.body
+ */
+/**
+ * @typedef SnapshotJson
+ * @property {'json'} responseType
+ * @property {string} fileSuffixKey
+ * @property {object} request
+ * @property {string} request.method
+ * @property {string} request.url
+ * @property {string[][]} request.headers
+ * @property {string|undefined} request.body
+ * @property {object} response
+ * @property {number} response.status
+ * @property {string} response.statusText
+ * @property {string[][]} response.headers
+ * @property {object|undefined} response.body
+ */
+
+/**
+ * @typedef {SnapshotText | SnapshotJson} Snapshot
+ */
+
+const identity = (response) => response;
+
+const defaultKeyDerivationProps = ['method', 'url', 'body'];
+async function defaultSnapshotFileNameGenerator(request) {
+  const url = new URL(request.url);
+  const filePrefix = [
+    request.method.toLowerCase(),
+    slugify(url.hostname),
+    slugify(url.pathname.replace('.json', '')),
+  ].filter(Boolean).join('-');
+
+  // Input data
+  const dataList = await Promise.all(
+    defaultKeyDerivationProps.map((key) => {
+      if (key === 'body') {
+        return request.clone().text();
+      }
+      return request[key];
+    }),
+  );
+
+  return {
+    filePrefix,
+    fileSuffixKey: dataList.join('#'),
+  };
+}
+
+// Dynamically changeable props
+/**
+ * @type {(response: Response, request: Request) => Promise<Response>}
+ */
+let responseTransformer = identity;
+/**
+ * @type {(req: Request) => Promise<{ filePrefix: string, fileSuffixKey: string }>}
+ */
+let snapshotFileNameGenerator = defaultSnapshotFileNameGenerator;
+
+/**
+ * @param {Request} request
+ */
+async function getSnapshotFileName(request) {
+  const { fileSuffixKey, filePrefix } = await snapshotFileNameGenerator(request.clone());
+
+  // 15 characters are enough for uniqueness
+  const hash = createHash('sha256')
+    .update(fileSuffixKey)
+    .digest('base64url')
+    .slice(0, 15);
+
+  const fileName = `${filePrefix}-${hash}.json`;
+
+  return {
+    absoluteFilePath: resolve(snapshotDirectory, fileName),
+    fileName,
+    filePrefix,
+    fileSuffixKey,
+  };
+}
+
+// NOTE: This isn't going to work on a test runner that uses multiple processes / workers
+const alreadyWrittenFiles = new Set();
+const readFiles = new Set();
+
+/**
+ * @param {Request} request
+ * @param {Response} response
+ */
+async function saveSnapshot(request, response) {
+  const { absoluteFilePath, fileName, fileSuffixKey } = await getSnapshotFileName(request);
+  // console.log(fileName);
+
+  // Prevent multiple tests from having same snapshot
+  if (alreadyWrittenFiles.has(absoluteFilePath)) return fileName;
+  alreadyWrittenFiles.add(absoluteFilePath);
+
+  let body;
+  /** @type {'text' | 'json'} */
+  let responseType;
+  const contentType = response.headers.get('content-type') || '';
+  if (contentType.includes('application/json') || contentType.includes('application/x-amz-json-1.0')) {
+    responseType = 'json';
+    body = await response.clone().json();
+  } else {
+    responseType = 'text';
+    body = await response.clone().text();
+  }
+  /** @type {Snapshot} */
+  const snapshot = {
+    request: {
+      method: request.method,
+      url: request.url,
+      headers: [...request.headers.entries()],
+      body: await request.clone().text(),
+    },
+    responseType,
+    response: {
+      status: response.status,
+      statusText: response.statusText,
+      headers: [...response.headers.entries()],
+      body,
+    },
+    fileSuffixKey,
+  };
+  const json = JSON.stringify(snapshot, null, 2);
+  await fs.writeFile(absoluteFilePath, json, 'utf-8');
+  return fileName;
+}
+
+const snapshotCache = {};
+/**
+ * @param {Request} request
+ */
+async function enforceSnapshotResponse(request) {
+  const { absoluteFilePath, fileName, fileSuffixKey } = await getSnapshotFileName(request);
+  // console.log(fileName);
+
+  if (!snapshotCache[absoluteFilePath]) {
+    let json;
+    try {
+      json = await fs.readFile(absoluteFilePath, 'utf-8');
+    } catch (err) {
+      // Fail any test that fires a real network request (without snapshot)
+      // @ts-ignore
+      if (err.code === 'ENOENT') {
+        const reqBody = await request.clone().text();
+        console.error('No network snapshot found for request with cache keys:', {
+          request: {
+            url: request.url,
+            method: request.method,
+            headers: Object.fromEntries([...request.headers.entries()]),
+            body: reqBody,
+          },
+          wouldBeFileSuffixKey: fileSuffixKey,
+          wouldBeFileName: fileName,
+        });
+        throw new Error('Network request not mocked');
+      } else {
+        // @ts-ignore
+        console.error('Error reading network snapshot file:', err.message);
+      }
+      return null;
+    }
+    snapshotCache[absoluteFilePath] = JSON.parse(json);
+    readFiles.add(fileName);
+  }
+
+  const snapshot = snapshotCache[absoluteFilePath];
+  const {
+    responseType,
+    response: {
+      status,
+      statusText,
+      headers,
+      body,
+    },
+  } = snapshot;
+
+  let newResponse = new Response(
+    responseType === 'json' ? JSON.stringify(body) : body,
+    {
+      status,
+      statusText,
+      headers: new Headers(/** @type HeadersInit */ (headers)),
+    },
+  );
+
+  newResponse = await responseTransformer(newResponse, request);
+
+  // respondWith is a method added by @mswjs/interceptors
+  // @ts-ignore
+  request.respondWith(newResponse);
+  return newResponse;
+}
+
+/**
+ * @type {import('@mswjs/interceptors').BatchInterceptor|null}
+ */
+let interceptor = null;
+
+let beforeExitEventSeen = false;
+let unusedFiles;
+process.on('beforeExit', async () => {
+  if (SNAPSHOT === 'read' && !beforeExitEventSeen) {
+    beforeExitEventSeen = true;
+    let files;
+    try {
+      files = await fs.readdir(snapshotDirectory);
+    } catch (err) {
+      return;
+    }
+    unusedFiles = files.filter((file) => !readFiles.has(file) && file !== unusedSnapshotsLogFile);
+    if (unusedFiles.length) {
+      await fs
+        .writeFile(
+          resolve(snapshotDirectory, unusedSnapshotsLogFile),
+          unusedFiles.join('\n'),
+          'utf-8',
+        )
+        .catch((err) => console.error(err));
+    } else {
+      await fs
+        .unlink(resolve(snapshotDirectory, unusedSnapshotsLogFile))
+        .catch((err) => {
+          if (err.code !== 'ENOENT') {
+            console.error(err);
+          }
+        });
+    }
+  }
+});
+
+// Attach snapshot filename generator function
+function attachSnapshotFilenameGenerator(func) {
+  snapshotFileNameGenerator = func;
+}
+
+// Reset snapshot filename generator to default
+function resetSnapshotFilenameGenerator() {
+  snapshotFileNameGenerator = defaultSnapshotFileNameGenerator;
+}
+
+// Attach response transformer function
+function attachResponseTransformer(func) {
+  responseTransformer = func;
+}
+
+// Reset response transformer
+function resetResponseTransformer() {
+  responseTransformer = identity;
+}
+
+// Start the interceptor
+function start({
+  snapshotDirectory: _snapshotDirectory = null,
+} = { snapshotDirectory: null }) {
+  if (!_snapshotDirectory) {
+    throw new Error('Please specify full path to a directory for storing/reading snapshots');
+  }
+  snapshotDirectory = _snapshotDirectory;
+  let dirCreatePromise;
+
+  interceptor = new BatchInterceptor({
+    name: 'http-snapshotter-interceptor',
+    interceptors: [
+      new ClientRequestInterceptor(),
+      new FetchInterceptor(),
+    ],
+  });
+
+  interceptor.on('request', async ({ request }) => {
+    if (SNAPSHOT === 'read') {
+      await enforceSnapshotResponse(request);
+    }
+  });
+  interceptor.on('response', async ({ request, response }) => {
+    if (SNAPSHOT === 'update') {
+      if (!dirCreatePromise) {
+        dirCreatePromise = fs.mkdir(snapshotDirectory, { recursive: true });
+      }
+      await dirCreatePromise;
+      saveSnapshot(request, response);
+    }
+    if (LOG_REQ) {
+      const { fileName, fileSuffixKey } = await getSnapshotFileName(request);
+      console.debug('Request', {
+        request: {
+          url: request.url,
+          method: request.method,
+          headers: Object.fromEntries([...request.headers.entries()]),
+          body: await request.clone().text(),
+        },
+        wouldBeFileName: fileName,
+        wouldBeFileSuffixKey: fileSuffixKey,
+      });
+    }
+  });
+  interceptor.apply();
+}
+
+// Stop the interceptor
+function stop() {
+  if (interceptor) {
+    interceptor.dispose();
+    interceptor = null;
+  }
+}
+
+// Singleton - as it makes sense only one interceptor be active at any given moment.
+module.exports = {
+  defaultSnapshotFileNameGenerator,
+  attachSnapshotFilenameGenerator,
+  resetSnapshotFilenameGenerator,
+  attachResponseTransformer,
+  resetResponseTransformer,
+  start,
+  stop,
+};

package/index.mjs

@@ -0,0 +1 @@
+export * from './index.js';

package/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "http-snapshotter",
+  "version": "0.1.0",
+  "description": "Snapshot HTTP requests for tests (node.js)",
+  "main": "index.cjs",
+  "exports": {
+    "import": "./index.mjs",
+    "require": "./index.js"
+  },
+  "scripts": {
+    "test": "tape tests/**/*.test.* | tap-diff"
+  },
+  "keywords": [
+    "snapshot",
+    "testing"
+  ],
+  "author": "Munawwar",
+  "license": "MIT",
+  "dependencies": {
+    "@mswjs/interceptors": "^0.24.1",
+    "@sindresorhus/slugify": "^1.1.2"
+  },
+  "devDependencies": {
+    "tap-diff": "^0.1.1",
+    "tape": "^5.6.6",
+    "typescript": "^5.2.2"
+  }
+}

package/tests/get-latest-xkcd-comic.test.js

@@ -0,0 +1,31 @@
+const test = require("tape");
+const { resolve } = require("node:path");
+
+const { start } = require("../index.js");
+
+const snapshotDirectory = resolve(__dirname, "http-snapshots");
+
+start({ snapshotDirectory });
+
+test("Latest XKCD comic (CJS)", async (t) => {
+  const res = await fetch("https://xkcd.com/info.0.json");
+  const json = await res.json();
+
+  t.deepEquals(
+    json,
+    {
+      month: "9",
+      num: 2829,
+      link: "",
+      year: "2023",
+      news: "",
+      safe_title: "Iceberg Efficiency",
+      transcript: "",
+      alt: "Our experimental aerogel iceberg with helium pockets manages true 100% efficiency, barely touching the water, and it can even lift off of the surface and fly to more efficiently pursue fleeing hubristic liners.",
+      img: "https://imgs.xkcd.com/comics/iceberg_efficiency.png",
+      title: "Iceberg Efficiency",
+      day: "15",
+    },
+    "must be deeply equal"
+  );
+});

package/tests/get-latest-xkcd-comic.test.mjs

@@ -0,0 +1,33 @@
+import test from "tape";
+import { fileURLToPath } from "node:url";
+import { resolve, dirname } from "node:path";
+import { start } from "../index.mjs";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const snapshotDirectory = resolve(__dirname, "http-snapshots");
+
+await start({ snapshotDirectory });
+
+test("Latest XKCD comic (ESM)", async (t) => {
+  const res = await fetch("https://xkcd.com/info.0.json");
+  const json = await res.json();
+
+  t.deepEquals(
+    json,
+    {
+      month: "9",
+      num: 2829,
+      link: "",
+      year: "2023",
+      news: "",
+      safe_title: "Iceberg Efficiency",
+      transcript: "",
+      alt: "Our experimental aerogel iceberg with helium pockets manages true 100% efficiency, barely touching the water, and it can even lift off of the surface and fly to more efficiently pursue fleeing hubristic liners.",
+      img: "https://imgs.xkcd.com/comics/iceberg_efficiency.png",
+      title: "Iceberg Efficiency",
+      day: "15",
+    },
+    "must be deeply equal"
+  );
+});

package/tests/http-snapshots/get-xkcd-com-info-0-arAlFb5gfcr9aCN.json

@@ -0,0 +1,101 @@
+{
+  "request": {
+    "method": "GET",
+    "url": "https://xkcd.com/info.0.json",
+    "headers": [],
+    "body": ""
+  },
+  "responseType": "json",
+  "response": {
+    "status": 200,
+    "statusText": "OK",
+    "headers": [
+      [
+        "accept-ranges",
+        "bytes"
+      ],
+      [
+        "age",
+        "228"
+      ],
+      [
+        "cache-control",
+        "max-age=300"
+      ],
+      [
+        "connection",
+        "keep-alive"
+      ],
+      [
+        "content-encoding",
+        "gzip"
+      ],
+      [
+        "content-length",
+        "287"
+      ],
+      [
+        "content-type",
+        "application/json"
+      ],
+      [
+        "date",
+        "Fri, 15 Sep 2023 18:26:59 GMT"
+      ],
+      [
+        "etag",
+        "W/\"65044c59-1c0\""
+      ],
+      [
+        "expires",
+        "Fri, 15 Sep 2023 12:28:00 GMT"
+      ],
+      [
+        "last-modified",
+        "Fri, 15 Sep 2023 12:21:45 GMT"
+      ],
+      [
+        "server",
+        "nginx"
+      ],
+      [
+        "vary",
+        "Accept-Encoding"
+      ],
+      [
+        "via",
+        "1.1 varnish, 1.1 varnish"
+      ],
+      [
+        "x-cache",
+        "HIT, HIT"
+      ],
+      [
+        "x-cache-hits",
+        "4250, 1"
+      ],
+      [
+        "x-served-by",
+        "cache-dfw-kdal2120106-DFW, cache-dxb1470033-DXB"
+      ],
+      [
+        "x-timer",
+        "S1694802419.108277,VS0,VE218"
+      ]
+    ],
+    "body": {
+      "month": "9",
+      "num": 2829,
+      "link": "",
+      "year": "2023",
+      "news": "",
+      "safe_title": "Iceberg Efficiency",
+      "transcript": "",
+      "alt": "Our experimental aerogel iceberg with helium pockets manages true 100% efficiency, barely touching the water, and it can even lift off of the surface and fly to more efficiently pursue fleeing hubristic liners.",
+      "img": "https://imgs.xkcd.com/comics/iceberg_efficiency.png",
+      "title": "Iceberg Efficiency",
+      "day": "15"
+    }
+  },
+  "fileSuffixKey": "GET#https://xkcd.com/info.0.json#"
+}