@tgwf/co2 0.5.0 → 0.8.0

package/.github/workflows/unittests.yml

@@ -0,0 +1,26 @@
+name: Unit tests
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+    branches:
+    - main
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [12.x, 14.x, 16.x]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v1
+      with:
+        node-version: ${{ matrix.node-version }}
+    - name: Install 
+      run: npm ci
+    - name: Verify lint
+      run: npm run lint
+    - name: Run unit tests
+      run: npm test

package/CHANGELOG.md

@@ -4,10 +4,60 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+-
 
-## [Unreleased]
+## Unreleased
 
-## [0.5.7] - 2020-03-03
+- Include a new alternative, "Green Byte Model" with the figures after speaking to folks at the IEA and other places.
+
+# [0.8.0] - 2021-11-28
+
+### Fixed
+
+- Update further dependencies
+- Fix embarassing order of magnitude typo in 1byte model (thanks @mstaschik!)
+
+## Added
+
+- Read JSON blob also as gzipped #44 (thanks @soulgalore)
+
+### Changed
+
+- The 1bye model will give different numbers now. It's mentioned in `#fixed` but it's worth repeating.
+
+## [0.7.0] - 2021-11-28
+
+### Fixed
+
+- Update tests to avoid network requests #50
+- Update dependencies across the board
+
+### Changed
+
+- Switch to github actions instead of travis for CI.
+
+## [0.6.1] - 2020-03-15
+
+### Fixed
+
+- Added the function to load JSON, on the tgwg.hosting object, for use in the sustaiable web sitespeed plugin.
+
+## [0.6.0] - 2020-03-15
+
+### Added
+
+- Added the hosting-JSON for running local checks against an array instead of SQLite.
+
+### Changed
+
+- Swapped out checking against a sqlite database `hosting-json`in favour of simple array in,
+- Updated conventions for style - using kebab-cases over CamelCase for naming files
+
+### Removed
+
+- Extracted sqlite usage and dependencies into a separate module, `url2green`. This means  you no longer need to compile SQLite on install.
+
+## [0.5.0] - 2020-03-03
 
 ### Changed
 

package/README.md

@@ -1,10 +1,11 @@
 # CO2
 
-<img src="https://travis-ci.org/thegreenwebfoundation/co2.js.svg?branch=master" />
+<img src="https://github.com/thegreenwebfoundation/co2.js/actions/workflows/unittests.yml/badge.svg" />
+
 
 We know computers use electricity, and because most of the electricity we use comes from burning fossil fuels to generate, there is an environmental cost to every upload and download we make over the internet.
 
-We cando something about this though. The same way we use performance budgets make apps and website faster and cheaper to run, we can use carbon budgets to make them faster, cheaper and _greener_.
+We can do something about this though. The same way we use performance budgets to make apps and websites faster and cheaper to run, we can use carbon budgets to make them faster, cheaper and _greener_.
 
 The CO2 package from [The Green Web Foundation][tgwf] lets you quickly estimate these emissions, to make measurable improvements as part of your workflow.
 
@@ -24,17 +25,40 @@ This is open source software, with all the guarantees associated, so if you want
 
 ## Usage
 
+### Calculating emissions per byte
+
+#### Server-side
+
+This approach relies on the `fs` module and so can only be used on platforms, like Node.js, that support this.
+
 ```js
 
 const CO2 = require('@tgwf/co2')
-const bytesSent = 1_000_000
+const bytesSent = (1024 * 1024 * 1024)
 const co2Emission = new CO2();
 estimatedCO2 = co2Emission.perByte(bytesSent)
 
-console.log(`Sending a million bytes, had a carbon footprint of ${estimatedCO2.toFixed(3)} grams of CO2`)
+console.log(`Sending a gigabyte, had a carbon footprint of ${estimatedCO2.toFixed(3)} grams of CO2`)
 
 ```
 
+#### Browser-side
+
+For browser-based solutions, you must import the `co2.js` submodule directly from `node_modules`. For example, like this:
+
+```js
+
+const CO2 = require('node_modules/@tgwf/co2/src/co2.js')
+const bytesSent = (1024 * 1024 * 1024)
+const co2Emission = new CO2();
+estimatedCO2 = co2Emission.perByte(bytesSent)
+
+console.log(`Sending a gigabyte, had a carbon footprint of ${estimatedCO2.toFixed(3)} grams of CO2`)
+
+```
+
+### Checking for green power
+
 Because different digital services and websites use different forms of power, there is also a module for checking if a domain uses green power or not, and whether the domains linked to on a page use green power as well.
 
 ```js
@@ -44,25 +68,30 @@ const greencheck = require('@tgwf/hosting')
 // returns true if green, otherwise false
 greencheck.check("google.com")
 
-// returns an array of the green domains, in this case ["google.
-greencheck.check(["google.com", "kochindustries.com"]) com"]
+// returns an array of the green domains, in this case ["google.com"].
+greencheck.check(["google.com", "kochindustries.com"])]
 
 // returns an array of green domains, again in this case, ["google.com"]
 greencheck.checkPage(["google.com"])
 
 ```
 
-Please note, we currently look at just the carbon cost of _generating_ the electricity, similar to how the IEA does, not the full life cycle cost of the energy, which would include things like:
+### Notes
+
+Please note, we currently look at just the carbon cost of _generating_ the electricity, similar to how the [International Energy Agency (IEA)] does, not the full life cycle cost of the energy.
+
+Doing this would include things like:
 
-- the carbon associated with digging up the fuel
-- the carbon associated with mining the materials to build the power stations, datacentres
+- the carbon emitted when carrying out activity associated with digging up the fuel
+- the carbon associated with mining the materials to _build_ the power stations, datacentres, and so on
 - the end of life costs
 - the maintenance costs over the life of the datacentres, power generation and end user devices, and the rest of the internet
 
-We'd like to do this, and we know how complicated this is likely to be.
+Life cycle figures do exist, but they are very difficult to do well. If you're interested in contributing to this, we'd love to hear from you.
 
 
 # Licenses
 
 Apache 2.0
 
+[International Energy Agency (IEA)]: https://www.iea.org/

package/data/fixtures/url2green.test.json

@@ -0,0 +1 @@
+["google.com","maxcdn.bootstrapcdn.com","thegreenwebfoundation.org","www.thegreenwebfoundation.org","fonts.googleapis.com","ajax.googleapis.com","assets.digitalclimatestrike.net","cdnjs.cloudflare.com","graphite.thegreenwebfoundation.org","analytics.thegreenwebfoundation.org","fonts.gstatic.com","api.thegreenwebfoundation.org"]

package/package.json

@@ -1,12 +1,13 @@
 {
   "name": "@tgwf/co2",
-  "version": "0.5.0",
+  "version": "0.8.0",
   "description": "Work out the co2 of your digital services",
   "main": "src/index.js",
   "scripts": {
     "test": "jest",
     "lint": "eslint .",
-    "lint:fix": "eslint . --fix"
+    "lint:fix": "eslint . --fix",
+    "travis": "npm run lint && jest"
   },
   "keywords": [
     "sustainability",
@@ -20,14 +21,17 @@
   "author": "Chris Adams",
   "license": "Apache-2.0",
   "devDependencies": {
-    "np": "^5.2.1",
-    "pagexray": "^2.5.9",
-    "jest": "25.1.0",
-    "eslint": "6.8.0",
-    "prettier": "1.19.1",
-    "eslint-config-prettier": "6.10.0",
-    "eslint-plugin-prettier": "3.1.2",
-    "eslint-plugin-jest": "23.8.1"
+    "@tgwf/url2green": "^0.4.0",
+    "eslint": "^8.3.0",
+    "eslint-config-prettier": "^8.3.0",
+    "eslint-plugin-jest": "^25.2.4",
+    "eslint-plugin-prettier": "^4.0.0",
+    "jest": "^27.3.1",
+    "minimist": "^1.2.5",
+    "nock": "^13.2.1",
+    "np": "^7.6.0",
+    "pagexray": "^4.3.1",
+    "prettier": "^2.4.1"
   },
   "np": {
     "yarn": false
@@ -36,7 +40,10 @@
     "access": "public"
   },
   "dependencies": {
-    "better-sqlite3": "^5.4.3",
     "debug": "^4.1.1"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/thegreenwebfoundation/co2.js.git"
   }
 }

package/src/1byte.js

@@ -9,19 +9,28 @@ const CO2_PER_KWH_IN_DC_GREY = 519;
 
 // The device usage figure combines figures for:
 //  1. the usage for devices (which is small proportion of the energy use)
-//  2. the *making* the device, which is comparitively high.
+//  2. the *making* the device, which is comparatively high.
 
-const KWH_PER_BYTE_IN_DC = 0.00000000072;
+const KWH_PER_BYTE_IN_DC = 7.2e-11;
 
 // this is probably best left as something users can define, or
 // a weighted average based on total usage.
+// Using a simple mean for now, as while web traffic to end users might trend
+// towards wifi and mobile,
+// Web traffic between servers is likely wired networks
+
+const FIXED_NETWORK_WIRED = 4.29e-10;
+const FIXED_NETWORK_WIFI = 1.52e-10;
+const FOUR_G_MOBILE = 8.84e-10;
+
 // Pull requests gratefully accepted
-const KWH_PER_BYTE_FOR_NETWORK = 0.00000000488;
+const KWH_PER_BYTE_FOR_NETWORK =
+  (FIXED_NETWORK_WIRED + FIXED_NETWORK_WIFI + FOUR_G_MOBILE) / 3;
 
-const KWH_PER_BYTE_FOR_DEVICES = 0.00000000013;
+const KWH_PER_BYTE_FOR_DEVICES = 1.3e-10;
 module.exports = {
   KWH_PER_BYTE_IN_DC,
   KWH_PER_BYTE_FOR_NETWORK,
   KWH_PER_BYTE_FOR_DEVICES,
-  CO2_PER_KWH_IN_DC_GREY
+  CO2_PER_KWH_IN_DC_GREY,
 };

package/src/1byte.test.js

@@ -0,0 +1,16 @@
+"use strict";
+
+const oneByte = require("./1byte");
+
+describe("onebyte", function () {
+  describe("perByte", function () {
+    it.only("returns a simple average of the different networks", function () {
+      // we limit this to 12 figures with toFixed(12), because
+      // we have a recurring 333333 afterwards
+      // 4.88e-10 is the same as 0.000000000488
+      const expected_val = (0.000000000488).toFixed(12);
+
+      expect(oneByte.KWH_PER_BYTE_FOR_NETWORK.toFixed(12)).toBe(expected_val);
+    });
+  });
+});

package/src/co2.js

@@ -57,10 +57,10 @@ class CO2 {
       co2PerDomain.push({
         domain,
         co2,
-        transferSize: pageXray.domains[domain].transferSize
+        transferSize: pageXray.domains[domain].transferSize,
       });
     }
-    co2PerDomain.sort(function(a, b) {
+    co2PerDomain.sort(function (a, b) {
       return b.co2 - a.co2;
     });
 
@@ -105,10 +105,10 @@ class CO2 {
       all.push({
         type,
         co2: co2PerContentType[type].co2,
-        transferSize: co2PerContentType[type].transferSize
+        transferSize: co2PerContentType[type].transferSize,
       });
     }
-    all.sort(function(a, b) {
+    all.sort(function (a, b) {
       return b.co2 - a.co2;
     });
     return all;
@@ -125,7 +125,7 @@ class CO2 {
       );
       allAssets.push({ url: asset.url, co2: co2ForTransfer, transferSize });
     }
-    allAssets.sort(function(a, b) {
+    allAssets.sort(function (a, b) {
       return b.co2 - a.co2;
     });
 

package/src/co2.test.js

@@ -6,137 +6,145 @@ const path = require("path");
 const CO2 = require("./co2");
 const pagexray = require("pagexray");
 
-describe("sustainableWeb", function() {
-  describe("co2", function() {
-    let har, co2;
-    const TGWF_GREY_VALUE = 2.0484539712;
-    const TGWF_GREEN_VALUE = 0.54704300112;
-    const TGWF_MIXED_VALUE = 1.6706517455999996;
+describe("co2", function () {
+  let har, co2;
+  const TGWF_GREY_VALUE = 0.20497;
+  const TGWF_GREEN_VALUE = 0.54704;
+  const TGWF_MIXED_VALUE = 0.16718;
 
-    const MILLION = 1000000;
-    const MILLION_GREY = 2.9064;
-    const MILLION_GREEN = 2.318;
+  const MILLION = 1000000;
+  const MILLION_GREY = 0.29081;
+  const MILLION_GREEN = 0.23196;
 
-    beforeEach(function() {
-      co2 = new CO2();
-      har = JSON.parse(
-        fs.readFileSync(
-          path.resolve(__dirname, "../data/fixtures/tgwf.har"),
-          "utf8"
-        )
+  beforeEach(function () {
+    co2 = new CO2();
+    har = JSON.parse(
+      fs.readFileSync(
+        path.resolve(__dirname, "../data/fixtures/tgwf.har"),
+        "utf8"
+      )
+    );
+  });
+
+  describe("perByte", function () {
+    it("returns a CO2 number for data transfer using 'grey' power", function () {
+      expect(co2.perByte(MILLION).toPrecision(5)).toBe(
+        MILLION_GREY.toPrecision(5)
       );
     });
 
-    describe("perByte", function() {
-      it("returns a CO2 number for data transfer using 'grey' power", function() {
-        expect(co2.perByte(MILLION)).toBe(MILLION_GREY);
-      });
-
-      it("returns a lower CO2 number for data transfer from domains using entirely 'green' power", function() {
-        expect(co2.perByte(MILLION, false)).toBe(MILLION_GREY);
-        expect(co2.perByte(MILLION, true)).toBe(MILLION_GREEN);
-      });
+    it("returns a lower CO2 number for data transfer from domains using entirely 'green' power", function () {
+      expect(co2.perByte(MILLION, false).toPrecision(5)).toBe(
+        MILLION_GREY.toPrecision(5)
+      );
+      expect(co2.perByte(MILLION, true).toPrecision(5)).toBe(
+        MILLION_GREEN.toPrecision(5)
+      );
     });
+  });
 
-    describe("perPage", function() {
-      it("returns CO2 for total transfer for page", function() {
-        const pages = pagexray.convert(har);
-        const pageXrayRun = pages[0];
+  describe("perPage", function () {
+    it("returns CO2 for total transfer for page", function () {
+      const pages = pagexray.convert(har);
+      const pageXrayRun = pages[0];
 
-        expect(co2.perPage(pageXrayRun)).toBe(TGWF_GREY_VALUE);
-      });
-      it("returns lower CO2 for page served from green site", function() {
-        const pages = pagexray.convert(har);
-        const pageXrayRun = pages[0];
-        let green = [
-          "www.thegreenwebfoundation.org",
-          "fonts.googleapis.com",
-          "ajax.googleapis.com",
-          "assets.digitalclimatestrike.net",
-          "cdnjs.cloudflare.com",
-          "graphite.thegreenwebfoundation.org",
-          "analytics.thegreenwebfoundation.org",
-          "fonts.gstatic.com",
-          "api.thegreenwebfoundation.org"
-        ];
-        expect(co2.perPage(pageXrayRun, green)).toBeLessThan(TGWF_GREY_VALUE);
-      });
-      it("returns a lower CO2 number where *some* domains use green power", function() {
-        const pages = pagexray.convert(har);
-        const pageXrayRun = pages[0];
-        // green can be true, or a array containing entries
-        let green = [
-          "www.thegreenwebfoundation.org",
-          "fonts.googleapis.com",
-          "ajax.googleapis.com",
-          "assets.digitalclimatestrike.net",
-          "cdnjs.cloudflare.com",
-          "graphite.thegreenwebfoundation.org",
-          "analytics.thegreenwebfoundation.org",
-          "fonts.gstatic.com",
-          "api.thegreenwebfoundation.org"
-        ];
-        expect(co2.perPage(pageXrayRun, green)).toBe(TGWF_MIXED_VALUE);
-      });
+      expect(co2.perPage(pageXrayRun).toPrecision(5)).toBe(
+        TGWF_GREY_VALUE.toPrecision(5)
+      );
+    });
+    it("returns lower CO2 for page served from green site", function () {
+      const pages = pagexray.convert(har);
+      const pageXrayRun = pages[0];
+      let green = [
+        "www.thegreenwebfoundation.org",
+        "fonts.googleapis.com",
+        "ajax.googleapis.com",
+        "assets.digitalclimatestrike.net",
+        "cdnjs.cloudflare.com",
+        "graphite.thegreenwebfoundation.org",
+        "analytics.thegreenwebfoundation.org",
+        "fonts.gstatic.com",
+        "api.thegreenwebfoundation.org",
+      ];
+      expect(co2.perPage(pageXrayRun, green)).toBeLessThan(TGWF_GREY_VALUE);
+    });
+    it("returns a lower CO2 number where *some* domains use green power", function () {
+      const pages = pagexray.convert(har);
+      const pageXrayRun = pages[0];
+      // green can be true, or a array containing entries
+      let green = [
+        "www.thegreenwebfoundation.org",
+        "fonts.googleapis.com",
+        "ajax.googleapis.com",
+        "assets.digitalclimatestrike.net",
+        "cdnjs.cloudflare.com",
+        "graphite.thegreenwebfoundation.org",
+        "analytics.thegreenwebfoundation.org",
+        "fonts.gstatic.com",
+        "api.thegreenwebfoundation.org",
+      ];
+      expect(co2.perPage(pageXrayRun, green).toPrecision(5)).toBe(
+        TGWF_MIXED_VALUE.toPrecision(5)
+      );
     });
-    describe("perDomain", function() {
-      it("shows object listing Co2 for each domain", function() {
-        const pages = pagexray.convert(har);
-        const pageXrayRun = pages[0];
-        const res = co2.perDomain(pageXrayRun);
+  });
+  describe("perDomain", function () {
+    it("shows object listing Co2 for each domain", function () {
+      const pages = pagexray.convert(har);
+      const pageXrayRun = pages[0];
+      const res = co2.perDomain(pageXrayRun);
 
-        const domains = [
-          "thegreenwebfoundation.org",
-          "www.thegreenwebfoundation.org",
-          "maxcdn.bootstrapcdn.com",
-          "fonts.googleapis.com",
-          "ajax.googleapis.com",
-          "assets.digitalclimatestrike.net",
-          "cdnjs.cloudflare.com",
-          "graphite.thegreenwebfoundation.org",
-          "analytics.thegreenwebfoundation.org",
-          "fonts.gstatic.com",
-          "api.thegreenwebfoundation.org"
-        ];
+      const domains = [
+        "thegreenwebfoundation.org",
+        "www.thegreenwebfoundation.org",
+        "maxcdn.bootstrapcdn.com",
+        "fonts.googleapis.com",
+        "ajax.googleapis.com",
+        "assets.digitalclimatestrike.net",
+        "cdnjs.cloudflare.com",
+        "graphite.thegreenwebfoundation.org",
+        "analytics.thegreenwebfoundation.org",
+        "fonts.gstatic.com",
+        "api.thegreenwebfoundation.org",
+      ];
 
-        for (let obj of res) {
-          expect(domains.indexOf(obj.domain)).toBeGreaterThan(-1);
-          expect(typeof obj.co2).toBe("number");
-        }
-      });
-      it("shows lower Co2 for green domains", function() {
-        const pages = pagexray.convert(har);
-        const pageXrayRun = pages[0];
+      for (let obj of res) {
+        expect(domains.indexOf(obj.domain)).toBeGreaterThan(-1);
+        expect(typeof obj.co2).toBe("number");
+      }
+    });
+    it("shows lower Co2 for green domains", function () {
+      const pages = pagexray.convert(har);
+      const pageXrayRun = pages[0];
 
-        const greenDomains = [
-          "www.thegreenwebfoundation.org",
-          "fonts.googleapis.com",
-          "ajax.googleapis.com",
-          "assets.digitalclimatestrike.net",
-          "cdnjs.cloudflare.com",
-          "graphite.thegreenwebfoundation.org",
-          "analytics.thegreenwebfoundation.org",
-          "fonts.gstatic.com",
-          "api.thegreenwebfoundation.org"
-        ];
-        const res = co2.perDomain(pageXrayRun);
-        const resWithGreen = co2.perDomain(pageXrayRun, greenDomains);
+      const greenDomains = [
+        "www.thegreenwebfoundation.org",
+        "fonts.googleapis.com",
+        "ajax.googleapis.com",
+        "assets.digitalclimatestrike.net",
+        "cdnjs.cloudflare.com",
+        "graphite.thegreenwebfoundation.org",
+        "analytics.thegreenwebfoundation.org",
+        "fonts.gstatic.com",
+        "api.thegreenwebfoundation.org",
+      ];
+      const res = co2.perDomain(pageXrayRun);
+      const resWithGreen = co2.perDomain(pageXrayRun, greenDomains);
 
-        for (let obj of res) {
-          expect(typeof obj.co2).toBe("number");
-        }
-        for (let obj of greenDomains) {
-          let index = 0;
-          expect(resWithGreen[index].co2).toBeLessThan(res[index].co2);
-          index++;
-        }
-      });
+      for (let obj of res) {
+        expect(typeof obj.co2).toBe("number");
+      }
+      for (let obj of greenDomains) {
+        let index = 0;
+        expect(resWithGreen[index].co2).toBeLessThan(res[index].co2);
+        index++;
+      }
     });
-    // describe('perContentType', function () {
-    //   test.skip('shows a breakdown of emissions by content type');
-    // });
-    // describe('dirtiestResources', function () {
-    //   it.skip('shows the top 10 resources by CO2 emissions');
-    // });
   });
+  // describe('perContentType', function () {
+  //   test.skip('shows a breakdown of emissions by content type');
+  // });
+  // describe('dirtiestResources', function () {
+  //   it.skip('shows the top 10 resources by CO2 emissions');
+  // });
 });

package/src/green-byte.js

@@ -22,5 +22,5 @@ module.exports = {
   KWH_PER_BYTE_IN_DC,
   KWH_PER_BYTE_FOR_NETWORK,
   KWH_PER_BYTE_FOR_DEVICES,
-  CO2_PER_KWH_IN_DC_GREY
+  CO2_PER_KWH_IN_DC_GREY,
 };

package/src/{hostingAPI.js → hosting-api.js}

@@ -28,7 +28,6 @@ async function checkDomainsAgainstAPI(domains) {
         )}`
       )
     );
-
     return greenDomainsFromResults(allGreenCheckResults);
   } catch (e) {
     return [];
@@ -37,20 +36,19 @@ async function checkDomainsAgainstAPI(domains) {
 
 function greenDomainsFromResults(greenResults) {
   const entries = Object.entries(greenResults);
-  let greenEntries = entries.filter(function([key, val]) {
+  let greenEntries = entries.filter(function ([key, val]) {
     return val.green;
   });
-
-  return greenEntries.map(function([key, val]) {
+  return greenEntries.map(function ([key, val]) {
     return val.url;
   });
 }
 
 async function getBody(url) {
   // Return new promise
-  return new Promise(function(resolve, reject) {
+  return new Promise(function (resolve, reject) {
     // Do async job
-    const req = https.get(url, function(res) {
+    const req = https.get(url, function (res) {
       if (res.statusCode < 200 || res.statusCode >= 300) {
         log(
           "Could not get info from the Green Web Foundation API, %s for %s",
@@ -61,7 +59,7 @@ async function getBody(url) {
       }
       const data = [];
 
-      res.on("data", chunk => {
+      res.on("data", (chunk) => {
         data.push(chunk);
       });
 
@@ -72,5 +70,5 @@ async function getBody(url) {
 }
 
 module.exports = {
-  check
+  check,
 };

package/src/hosting-api.test.js

@@ -0,0 +1,37 @@
+"use strict";
+
+const hosting = require("./hosting-api");
+const nock = require("nock");
+
+describe("hostingAPI", function () {
+  describe("checking a single domain with #check", function () {
+    it("using the API", async function () {
+      const scope = nock("https://api.thegreenwebfoundation.org/")
+        .get("/greencheck/google.com")
+        .reply(200, {
+          url: "google.com",
+          green: true,
+        });
+      const res = await hosting.check("google.com");
+      expect(res).toEqual(true);
+    });
+  });
+  describe("implicitly checking multiple domains with #check", function () {
+    it("using the API", async function () {
+      const scope = nock("https://api.thegreenwebfoundation.org/")
+        .get("/v2/greencheckmulti/[%22google.com%22,%22kochindustries.com%22]")
+        .reply(200, {
+          "google.com": {
+            url: "google.com",
+            green: true,
+          },
+          "kochindustries.com": {
+            url: "kochindustries.com",
+            green: null,
+          },
+        });
+      const res = await hosting.check(["google.com", "kochindustries.com"]);
+      expect(res).toContain("google.com");
+    });
+  });
+});

package/src/hosting-database.test.js

@@ -0,0 +1,31 @@
+"use strict";
+
+const log = require("debug")("tgwf:url2green:test");
+const { hosting } = require("@tgwf/url2green");
+const path = require("path");
+
+const dbPath = path.resolve(
+  __dirname,
+  "..",
+  "data",
+  "fixtures",
+  "url2green.test.db"
+);
+
+describe("hostingDatabase", function () {
+  describe("checking a single domain with #check", function () {
+    test("tries to use a local database if available ", async function () {
+      const res = await hosting.check("google.com", dbPath);
+      expect(res).toEqual(true);
+    });
+  });
+  describe("implicitly checking multiple domains with #check", function () {
+    test("tries to use a local database if available", async function () {
+      const res = await hosting.check(
+        ["google.com", "kochindustries.com"],
+        dbPath
+      );
+      expect(res).toContain("google.com");
+    });
+  });
+});

package/src/hosting-json.js

@@ -0,0 +1,75 @@
+"use strict";
+
+const log = require("debug")("tgwf:hostingCache");
+const path = require("path");
+const fs = require("fs");
+const zlib = require("zlib");
+const { promisify } = require("util");
+const readFile = promisify(fs.readFile);
+const gunzip = promisify(zlib.gunzip);
+
+async function streamToString(stream) {
+  return new Promise((resolve, reject) => {
+    const chunks = [];
+    stream.on("error", reject);
+    stream.on("data", (chunk) => chunks.push(chunk));
+    stream.on("end", () => resolve(Buffer.concat(chunks)));
+  });
+}
+
+async function getGzippedFileAsJson(jsonPath) {
+  const readStream = fs.createReadStream(jsonPath);
+  const text = await streamToString(readStream);
+  const unzipped = await gunzip(text);
+  return unzipped.toString();
+}
+
+async function loadJSON(jsonPath) {
+  const jsonBuffer = jsonPath.endsWith(".gz")
+    ? await getGzippedFileAsJson(jsonPath)
+    : await readFile(jsonPath);
+  return JSON.parse(jsonBuffer);
+}
+
+async function check(domain, db) {
+  // is it a single domain or an array of them?
+  if (typeof domain === "string") {
+    return checkInJSON(domain, db);
+  } else {
+    return checkDomainsInJSON(domain, db);
+  }
+}
+
+function checkInJSON(domain, db) {
+  if (db.indexOf(domain) > -1) {
+    return true;
+  }
+  return false;
+}
+
+function greenDomainsFromResults(greenResults) {
+  const entries = Object.entries(greenResults);
+  let greenEntries = entries.filter(function ([key, val]) {
+    return val.green;
+  });
+
+  return greenEntries.map(function ([key, val]) {
+    return val.url;
+  });
+}
+
+function checkDomainsInJSON(domains, db) {
+  let greenDomains = [];
+
+  for (let domain of domains) {
+    if (db.indexOf(domain) > -1) {
+      greenDomains.push(domain);
+    }
+  }
+  return greenDomains;
+}
+
+module.exports = {
+  check,
+  loadJSON,
+};

package/src/hosting-json.test.js

@@ -0,0 +1,44 @@
+"use strict";
+
+const hosting = require("./hosting-json");
+const path = require("path");
+
+describe("hostingJSON", function () {
+  const jsonPath = path.resolve(
+    __dirname,
+    "..",
+    "data",
+    "fixtures",
+    "url2green.test.json"
+  );
+  const jsonPathGz = path.resolve(
+    __dirname,
+    "..",
+    "data",
+    "fixtures",
+    "url2green.test.json.gz"
+  );
+  describe("checking a single domain with #check", function () {
+    test("against the list of domains as JSON", async function () {
+      const db = await hosting.loadJSON(jsonPath);
+      const res = await hosting.check("google.com", db);
+      expect(res).toEqual(true);
+    });
+  });
+  describe("checking a single domain with #check", function () {
+    test("against the list of domains as JSON loaded from a gzipped JSON", async function () {
+      const db = await hosting.loadJSON(jsonPathGz);
+      const res = await hosting.check("google.com", db);
+      expect(res).toEqual(true);
+    });
+  });
+  describe("implicitly checking multiple domains with #check", function () {
+    test("against the list of domains as JSON", async function () {
+      const db = await hosting.loadJSON(jsonPath);
+      const domains = ["google.com", "kochindustries.com"];
+
+      const res = await hosting.check(domains, db);
+      expect(res).toContain("google.com");
+    });
+  });
+});

package/src/hosting.js

@@ -1,12 +1,12 @@
 "use strict";
 
-// const log = require("debug")("tgwf:hosting");
-const hostingAPI = require("./hostingAPI");
-const hostingDatabase = require("./hostingDatabase");
+const log = require("debug")("tgwf:hosting");
+const hostingAPI = require("./hosting-api");
+const hostingJSON = require("./hosting-json");
 
-function check(domain, dbName) {
-  if (dbName) {
-    return hostingDatabase.check(domain, dbName);
+function check(domain, db) {
+  if (db) {
+    return hostingJSON.check(domain, db);
   } else {
     return hostingAPI.check(domain);
   }
@@ -14,22 +14,23 @@ function check(domain, dbName) {
 
 function greenDomainsFromResults(greenResults) {
   const entries = Object.entries(greenResults);
-  let greenEntries = entries.filter(function([key, val]) {
+  let greenEntries = entries.filter(function ([key, val]) {
     return val.green;
   });
 
-  return greenEntries.map(function([key, val]) {
+  return greenEntries.map(function ([key, val]) {
     return val.url;
   });
 }
 
-async function checkPage(pageXray) {
+async function checkPage(pageXray, db) {
   const domains = Object.keys(pageXray.domains);
-  return check(domains);
+  return check(domains, db);
 }
 
 module.exports = {
   check,
   checkPage,
-  greenDomains: greenDomainsFromResults
+  greenDomains: greenDomainsFromResults,
+  loadJSON: hostingJSON.loadJSON,
 };

package/src/hosting.test.js

@@ -6,9 +6,17 @@ const path = require("path");
 const hosting = require("./hosting");
 const pagexray = require("pagexray");
 
-describe("hosting", function() {
+const jsonPath = path.resolve(
+  __dirname,
+  "..",
+  "data",
+  "fixtures",
+  "url2green.test.json"
+);
+
+describe("hosting", function () {
   let har;
-  beforeEach(function() {
+  beforeEach(function () {
     har = JSON.parse(
       fs.readFileSync(
         path.resolve(__dirname, "../data/fixtures/tgwf.har"),
@@ -16,17 +24,16 @@ describe("hosting", function() {
       )
     );
   });
-  describe("checking all domains on a page object with #checkPage ", function() {
-    it("it returns a list of green domains, when passed a page object", async function() {
+  describe("checking all domains on a page object with #checkPage ", function () {
+    it("it returns a list of green domains, when passed a page object", async function () {
       const pages = pagexray.convert(har);
       const pageXrayRun = pages[0];
+      const db = await hosting.loadJSON(jsonPath);
+      const greenDomains = await hosting.checkPage(pageXrayRun, db);
 
-      // TODO find a way to not hit the API each time
-      const greenDomains = await hosting.checkPage(pageXrayRun);
-
-      expect(greenDomains).toHaveLength(10);
-
+      expect(greenDomains).toHaveLength(11);
       const expectedGreendomains = [
+        "maxcdn.bootstrapcdn.com",
         "thegreenwebfoundation.org",
         "www.thegreenwebfoundation.org",
         "fonts.googleapis.com",
@@ -36,9 +43,9 @@ describe("hosting", function() {
         "graphite.thegreenwebfoundation.org",
         "analytics.thegreenwebfoundation.org",
         "fonts.gstatic.com",
-        "api.thegreenwebfoundation.org"
+        "api.thegreenwebfoundation.org",
       ];
-      greenDomains.forEach(function(dom) {
+      greenDomains.forEach(function (dom) {
         expect(expectedGreendomains).toContain(dom);
       });
     });
@@ -46,42 +53,25 @@ describe("hosting", function() {
     //   'it returns an empty list, when passed a page object with no green domains'
     // );
   });
-  describe("checking a single domain with #check", function() {
-    it("tries to use a local database", async function() {
-      const res = await hosting.check(
-        "google.com",
-        path.resolve(__dirname, "..", "url2green.test.db")
-      );
-      expect(res).toEqual(true);
-    });
-    it("use the API instead", async function() {
-      const res = await hosting.check("google.com");
+  describe("checking a single domain with #check", function () {
+    it("use the API instead", async function () {
+      const db = await hosting.loadJSON(jsonPath);
+      const res = await hosting.check("google.com", db);
       expect(res).toEqual(true);
     });
   });
-  describe("implicitly checking multiple domains with #check", function() {
-    it("tries to use a local database if available", async function() {
-      const res = await hosting.check(
-        ["google.com", "kochindustries.com"],
-        path.resolve(__dirname, "..", "url2green.test.db")
-      );
-      expect(res).toContain("google.com");
-    });
-    it("Use the API", async function() {
-      const res = await hosting.check(["google.com", "kochindustries.com"]);
+  describe("implicitly checking multiple domains with #check", function () {
+    it("Use the API", async function () {
+      const db = await hosting.loadJSON(jsonPath);
+
+      const res = await hosting.check(["google.com", "kochindustries.com"], db);
       expect(res).toContain("google.com");
     });
   });
-  describe("explicitly checking multiple domains with #checkMulti", function() {
-    it("tries to use a local database if available", async function() {
-      const res = await hosting.check(
-        ["google.com", "kochindustries.com"],
-        path.resolve(__dirname, "..", "url2green.test.db")
-      );
-      expect(res).toContain("google.com");
-    });
-    it("use the API", async function() {
-      const res = await hosting.check(["google.com", "kochindustries.com"]);
+  describe("explicitly checking multiple domains with #checkMulti", function () {
+    it("use the API", async function () {
+      const db = await hosting.loadJSON(jsonPath);
+      const res = await hosting.check(["google.com", "kochindustries.com"], db);
       expect(res).toContain("google.com");
     });
   });

package/src/index.js

@@ -3,5 +3,5 @@ const hosting = require("./hosting");
 
 module.exports = {
   co2,
-  hosting
+  hosting,
 };

package/.travis.yml

@@ -1,11 +0,0 @@
-language: node_js
-node_js: 10
-matrix:
-  include:
-    - name: "Test on Linux"
-      dist: bionic
-    - name: "Test on OS X"
-      os: osx
-      osx_image: xcode11.3  
-    - name: "Test on Windows"
-      os: windows

package/src/hostingAPI.test.js

@@ -1,18 +0,0 @@
-"use strict";
-
-const hosting = require("./hostingAPI");
-
-describe("hostingAPI", function() {
-  describe("checking a single domain with #check", function() {
-    it("using the API", async function() {
-      const res = await hosting.check("google.com");
-      expect(res).toEqual(true);
-    });
-  });
-  describe("implicitly checking multiple domains with #check", function() {
-    it("using the API", async function() {
-      const res = await hosting.check(["google.com", "kochindustries.com"]);
-      expect(res).toContain("google.com");
-    });
-  });
-});

package/src/hostingDatabase.js

@@ -1,80 +0,0 @@
-"use strict";
-
-const log = require("debug")("tgwf:hostingDatabase");
-const Database = require("better-sqlite3");
-
-function getQ(domains) {
-  const q = [];
-  for (let domain of domains) {
-    q.push("?");
-  }
-  return q.join(",");
-}
-
-function getDatabase(databaseFullPathAndName) {
-  log(`looking for db at ${databaseFullPathAndName}`);
-  return new Database(databaseFullPathAndName, {
-    readonly: true,
-    fileMustExist: true
-  });
-}
-
-function check(domain, dbName) {
-  let db;
-
-  try {
-    db = getDatabase(dbName);
-  } catch (SqliteError) {
-    log(`couldn't find SQlite database at path: ${dbName}`);
-    throw SqliteError;
-  }
-
-  // is it a single domain or an array of them?
-  if (typeof domain === "string") {
-    return checkInDB(domain, db);
-  } else {
-    return checkDomainsInDB(domain, db);
-  }
-}
-
-function checkInDB(domain, db) {
-  try {
-    const stmt = db.prepare("SELECT * FROM green_presenting WHERE url = ?");
-    return !!stmt.get(domain).green;
-  } finally {
-    if (db) {
-      db.close();
-    }
-  }
-}
-
-function greenDomainsFromResults(greenResults) {
-  const entries = Object.entries(greenResults);
-  let greenEntries = entries.filter(function([key, val]) {
-    return val.green;
-  });
-
-  return greenEntries.map(function([key, val]) {
-    return val.url;
-  });
-}
-
-function checkDomainsInDB(domains, db) {
-  try {
-    const stmt = db.prepare(
-      `SELECT * FROM green_presenting WHERE url in (${getQ(domains)})`
-    );
-
-    const res = stmt.all(domains);
-
-    return greenDomainsFromResults(res);
-  } finally {
-    if (db) {
-      db.close();
-    }
-  }
-}
-
-module.exports = {
-  check
-};

package/src/hostingDatabase.test.js

@@ -1,25 +0,0 @@
-"use strict";
-
-const hosting = require("./hostingDatabase");
-const path = require("path");
-
-describe("hostingDatabase", function() {
-  describe("checking a single domain with #check", function() {
-    it("tries to use a local database if available ", async function() {
-      const res = await hosting.check(
-        "google.com",
-        path.resolve(__dirname, "..", "url2green.test.db")
-      );
-      expect(res).toEqual(true);
-    });
-  });
-  describe("implicitly checking multiple domains with #check", function() {
-    it("tries to use a local database if available", async function() {
-      const res = await hosting.check(
-        ["google.com", "kochindustries.com"],
-        path.resolve(__dirname, "..", "url2green.test.db")
-      );
-      expect(res).toContain("google.com");
-    });
-  });
-});