builtwith-api 1.0.5 → 1.0.8

package/README.md

@@ -2,6 +2,15 @@
 
 `builtwith-api` is a utility wrapper for the BuiltWith API suite.
 
+Install using
+```
+yarn install builtwith-api
+
+or
+
+npm install builtwith-api
+```
+
 ## Features
 
 Response Formats
@@ -18,13 +27,14 @@ APIS
 - trends
 - companyToUrl
 - domainLive
+- trust
 
 ________________
 
 ## How To Use
 
 ```js
-const builtwith = require('builtwith')
+const BuiltWith = require('builtwith-api')
 
 const builtwith = BuiltWith(process.env.YOUR_BUILTWITH_API_KEY, {
   responseFormat: 'json' // 'json' 'xml' 'txt' (only for lists API)
@@ -60,7 +70,9 @@ await builtwith.lists(technology, {
 
 await builtwith.relationships(url)
 
-await builtwith.keywords(url)
+const urls = ['hotelscombined.com', 'builtwith.com']
+// Multi-domain lookup. Will automatically be converted into URI encoded array (hotelscombined.com,builtwith.com).
+await builtwith.keywords(urls)
 
 await builtwith.trends(technology, {
   // Totals will be the closest to this date - providing the ability to get historical totals
@@ -76,6 +88,13 @@ await builtwith.companyToUrl(companyName, {
 })
 
 await builtwith.domainLive(url)
+
+await builtwith.trust(url, {
+  // If the words specified here are in the HTML of the website the result will have Stopwords set to true for LIVE lookups.
+  words: 'medicine,masks',
+  // Performs a live lookup of the website in question. This slows down the response. A result with a status of 'needLive' requires the LIVE option to determine if the website is suspect or not. Live lookups slow down the response of the API, you should consider calling this if the non-LIVE lookup response status is 'needLive'.
+  live: false
+})
 ```
 
 Made live on Facebook by Zach Caceres during Covid-19 Quarantine

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "builtwith-api",
-  "version": "1.0.5",
+  "version": "1.0.8",
   "description": "Wrapper for the BuiltWith API",
   "main": "src/index.js",
   "scripts": {},
@@ -14,11 +14,14 @@
     "BuiltWith",
     "apis"
   ],
+  "files": [
+    "src",
+    "src/utils"
+  ],
   "author": "Zach Caceres (@zachcaceres)",
   "license": "MIT",
   "dependencies": {
-    "lodash": "^4.17.15",
-    "node-fetch": "^2.6.0"
+    "lodash": "^4.17.15"
   },
   "devDependencies": {
     "dotenv": "^8.2.0"

package/src/index.js

@@ -1,34 +1,44 @@
-const _ = require('lodash')
-
-const utils = require("./utils")
-const { VALID_RESPONSE_TYPES } = require("./config")
+const _ = require("lodash");
 
+const utils = require("./utils");
+const { VALID_RESPONSE_TYPES } = require("./config");
 
 function BuiltWith(apiKey, moduleParams = {}) {
-  const responseFormat = _.get(moduleParams, 'responseFormat', 'json')
+  const responseFormat = _.get(moduleParams, "responseFormat", "json");
 
   if (!Object.values(VALID_RESPONSE_TYPES).includes(responseFormat)) {
-    throw new Error(`Invalid 'responseFormat'. Valid format are 'xml', 'txt', and 'json'. You input ${responseFormat}`)
+    throw new Error(
+      `Invalid 'responseFormat'. Valid format are 'xml', 'txt', and 'json'. You input ${responseFormat}`,
+    );
   }
 
   if (responseFormat === VALID_RESPONSE_TYPES.TXT) {
-    console.warn("TXT response format is only supported for the BuiltWith Lists API. See: https://api.builtwith.com/lists-api");
+    console.warn(
+      "TXT response format is only supported for the BuiltWith Lists API. See: https://api.builtwith.com/lists-api",
+    );
   }
 
   function constructBuiltWithURL(
     apiName,
     requestParams = {},
-    subdomain = 'api'
+    subdomain = "api",
   ) {
-    let bwURL = `https://${subdomain}.builtwith.com/${apiName}/api.${responseFormat}?KEY=${apiKey}`
+    let bwURL = `https://${subdomain}.builtwith.com/${apiName}/api.${responseFormat}?KEY=${apiKey}`;
 
     if (!_.isEmpty(requestParams)) {
-      bwURL += `&${utils.paramsObjToQueryString(requestParams)}`
+      bwURL += `&${utils.paramsObjToQueryString(requestParams)}`;
     }
 
-    return bwURL
+    return bwURL;
   }
 
+  function checkUrlData(url, isMultiDomain = true) {
+    let isArray = Array.isArray(url);
+    if (isArray) {
+      if (!isMultiDomain) throw "API does not allow for multi-domain LOOKUP";
+      if (url.length > 16) throw "Domain LOOKUP size to big (16 max)";
+    }
+  }
 
   return {
     /**
@@ -37,12 +47,14 @@ function BuiltWith(apiKey, moduleParams = {}) {
      * @see https://api.builtwith.com/free-api
      * @param {String} url
      */
-    free: async function(url) {
+    free: async function (url) {
+      checkUrlData(url, false);
+
       const bwURL = constructBuiltWithURL("free1", {
-        LOOKUP: url
+        LOOKUP: url,
       });
 
-      const res = await utils.makeStandardRequest(bwURL, responseFormat)
+      const res = await utils.makeStandardRequest(bwURL, responseFormat);
       return res;
     },
 
@@ -50,27 +62,28 @@ function BuiltWith(apiKey, moduleParams = {}) {
      * Make a request to the BuiltWith Domain API
      *
      * @see https://api.builtwith.com/domain-api
-     * @param {String} url
+     * @param {(string|string[])} url
      * @param {Object} params
      */
-    domain: async function(url, params) {
+    domain: async function (url, params) {
+      checkUrlData(url);
       const hideAll = _.get(params, "hideAll", false);
       const noMetaData = _.get(params, "noMetaData", false);
       const noAttributeData = _.get(params, "noAttributeData", false);
       const hideDescriptionAndLinks = _.get(
         params,
         "hideDescriptionAndLinks",
-        false
+        false,
       );
       const onlyLiveTechnologies = _.get(params, "onlyLiveTechnologies", false);
 
-      const bwURL = constructBuiltWithURL("v14", {
+      const bwURL = constructBuiltWithURL("v15", {
         LOOKUP: url,
         HIDETEXT: hideAll,
         HIDEDL: hideDescriptionAndLinks,
         LIVEONLY: onlyLiveTechnologies,
         NOMETA: noMetaData,
-        NOATTR: noAttributeData
+        NOATTR: noAttributeData,
       });
 
       const res = await utils.makeStandardRequest(bwURL, responseFormat);
@@ -85,7 +98,7 @@ function BuiltWith(apiKey, moduleParams = {}) {
      * @param {String} technology
      * @param {Object} params
      */
-    lists: async function(technology, params) {
+    lists: async function (technology, params) {
       const includeMetaData = _.get(params, "includeMetaData", false);
       const offset = _.get(params, "offset");
       const since = _.get(params, "since");
@@ -94,11 +107,11 @@ function BuiltWith(apiKey, moduleParams = {}) {
         TECH: technology,
         META: includeMetaData,
         OFFSET: offset,
-        SINCE: since
+        SINCE: since,
       });
 
-      const res = await utils.makeBulletProofRequest(bwURL)
-      return res
+      const res = await utils.makeBulletProofRequest(bwURL);
+      return res;
     },
 
     /**
@@ -107,11 +120,12 @@ function BuiltWith(apiKey, moduleParams = {}) {
      * @note Relationships API may throw an error related to the maxJSONLength property for certain URLs. This is thrown in BuiltWith and cannot be handled here.
      *
      * @see https://api.builtwith.com/relationships-api
-     * @param {String} url
+     * @param {(string|string[])} url
      */
-    relationships: async function(url) {
+    relationships: async function (url) {
+      checkUrlData(url);
       const bwURL = constructBuiltWithURL("rv1", {
-        LOOKUP: url
+        LOOKUP: url,
       });
 
       const res = await utils.makeStandardRequest(bwURL, responseFormat);
@@ -123,14 +137,15 @@ function BuiltWith(apiKey, moduleParams = {}) {
      * Make a request to the BuiltWith Keywords API
      *
      * @see https://api.builtwith.com/keywords-api
-     * @param {String} url
+     * @param {(string|string[])} url
      */
-    keywords: async function(url) {
+    keywords: async function (url) {
+      checkUrlData(url);
       const bwURL = constructBuiltWithURL("kw2", {
-        LOOKUP: url
+        LOOKUP: url,
       });
 
-      const res = await utils.makeStandardRequest(bwURL, responseFormat)
+      const res = await utils.makeStandardRequest(bwURL, responseFormat);
       return res;
     },
 
@@ -141,16 +156,16 @@ function BuiltWith(apiKey, moduleParams = {}) {
      * @param {String} technology
      * @param {Object} params
      */
-    trends: async function(technology, params) {
+    trends: async function (technology, params) {
       const date = _.get(params, "date");
 
       const bwURL = constructBuiltWithURL("trends/v6", {
         TECH: technology,
-        DATE: date
+        DATE: date,
       });
 
       const res = await utils.makeBulletProofRequest(bwURL);
-      return res
+      return res;
     },
 
     /**
@@ -160,7 +175,7 @@ function BuiltWith(apiKey, moduleParams = {}) {
      * @param {String} companyName
      * @param {Object} params
      */
-    companyToUrl: async function(companyName, params) {
+    companyToUrl: async function (companyName, params) {
       const tld = _.get(params, "tld");
       const amount = _.get(params, "noMetaData");
 
@@ -169,12 +184,12 @@ function BuiltWith(apiKey, moduleParams = {}) {
         {
           COMPANY: encodeURIComponent(companyName),
           TLD: tld,
-          AMOUNT: amount
+          AMOUNT: amount,
         },
-        "ctu"
+        "ctu",
       );
 
-      const res = await utils.makeStandardRequest(bwURL, responseFormat)
+      const res = await utils.makeStandardRequest(bwURL, responseFormat);
       return res;
     },
 
@@ -184,21 +199,46 @@ function BuiltWith(apiKey, moduleParams = {}) {
      * @see https://api.builtwith.com/domain-live-api
      * @param {String} url
      */
-    domainLive: async function(url) {
-      const bwURL = constructBuiltWithURL("dlv1", {
-        LOOKUP: url
+    domainLive: async function (url) {
+      const bwURL = constructBuiltWithURL("ddlv2", {
+        LOOKUP: url,
       });
 
-      const res = await utils.makeStandardRequest(bwURL, responseFormat)
+      const res = await utils.makeStandardRequest(bwURL, responseFormat);
       return res;
-    }
+    },
+
+    /**
+     * Make a request to the BuiltWith Domain Live API
+     *
+     * @see https://api.builtwith.com/trust-api
+     * @param {String} url
+     * @param {Object} params
+     */
+    trust: async function (url, params) {
+      const words = _.get(params, "words", "");
+      const live = _.get(params, "live", false);
+
+      const bwURL = constructBuiltWithURL("trustv1", {
+        LOOKUP: url,
+        // 'wordOne, wordTwo' ==> 'wordOne,wordTwo'
+        WORDS: words
+          .split(",")
+          .map((wrd) => wrd.trim())
+          .join(","),
+        LIVE: live,
+      });
+
+      const res = await utils.makeStandardRequest(bwURL, responseFormat);
+      return res;
+    },
   };
 }
 
 // Constructor to authenticate and get module
-module.exports = function(apiKey, moduleParams) {
+module.exports = function (apiKey, moduleParams) {
   if (!apiKey) {
-    throw new Error('You must initialize the BuiltWith module with an api key')
+    throw new Error("You must initialize the BuiltWith module with an api key");
   }
-  return BuiltWith(apiKey, moduleParams)
-}
+  return BuiltWith(apiKey, moduleParams);
+};

package/src/index.test.js

@@ -0,0 +1,90 @@
+const nock = require('nock');
+const BuiltWith = require('../src/index');
+
+const apiKey = 'testApiKey';
+const moduleParams = { isBulletProof: false };
+
+describe('BuiltWith', () => {
+  beforeEach(() => {
+    nock('https://api.builtwith.com')
+      .get(/.*/)
+      .reply(200, { test: 'data' });
+  });
+
+  afterEach(() => {
+    nock.cleanAll();
+  });
+
+  it('should make a standard request for the domain API', async () => {
+    const bw = BuiltWith(apiKey, moduleParams);
+
+    const response = await bw.domain('testUrl');
+
+    expect(response).toEqual({ test: 'data' });
+  });
+
+  it('should make a standard request for the lists API', async () => {
+    const bw = BuiltWith(apiKey, moduleParams);
+
+    const response = await bw.lists('testUrl');
+
+    expect(response).toEqual({ test: 'data' });
+  });
+
+  it('should make a standard request for the relationships API', async () => {
+    const bw = BuiltWith(apiKey, moduleParams);
+
+    const response = await bw.relationships('testUrl');
+
+    expect(response).toEqual({ test: 'data' });
+  });
+
+
+  it('should make a standard request for the free API', async () => {
+    const bw = BuiltWith(apiKey, moduleParams);
+
+    const response = await bw.free('testUrl');
+
+    expect(response).toEqual({ test: 'data' });
+  });
+
+  it('should make a standard request for the keywords API', async () => {
+    const bw = BuiltWith(apiKey, moduleParams);
+
+    const response = await bw.keywords('testUrl');
+
+    expect(response).toEqual({ test: 'data' });
+  });
+
+  it('should make a standard request for the trends API', async () => {
+    const bw = BuiltWith(apiKey, moduleParams);
+
+    const response = await bw.trends('testUrl');
+
+    expect(response).toEqual({ test: 'data' });
+  });
+
+  it('should make a standard request for the companyToUrl API', async () => {
+    const bw = BuiltWith(apiKey, moduleParams);
+
+    const response = await bw.companyToUrl('My Company');
+
+    expect(response).toEqual({ test: 'data' });
+  });
+
+  it('should make a standard request for the domainLive API', async () => {
+    const bw = BuiltWith(apiKey, moduleParams);
+
+    const response = await bw.domainLive('testUrl');
+
+    expect(response).toEqual({ test: 'data' });
+  });
+
+  it('should make a standard request for the trust API', async () => {
+    const bw = BuiltWith(apiKey, moduleParams);
+
+    const response = await bw.trust('testUrl');
+
+    expect(response).toEqual({ test: 'data' });
+  });
+});

package/src/utils.js

@@ -1,18 +1,16 @@
-const fetch = require("node-fetch")
-const { VALID_RESPONSE_TYPES } = require("./config")
+const { VALID_RESPONSE_TYPES } = require("./config");
 
 module.exports = {
-  makeStandardRequest: async function(url, responseFormat) {
-    return fetch(url).then(res => {
-      if (responseFormat === VALID_RESPONSE_TYPES.XML) {
-        return res.text();
-      } else {
-        return res.json();
-      }
-    });
+  makeStandardRequest: async function (url, responseFormat) {
+    const res = await fetch(url);
+    if (responseFormat === VALID_RESPONSE_TYPES.XML) {
+      return res.text();
+    } else {
+      return res.json();
+    }
   },
 
-  makeBulletProofRequest: async function(url, responseFormat) {
+  makeBulletProofRequest: async function (url, responseFormat) {
     const res = await fetch(url);
     if (
       responseFormat === VALID_RESPONSE_TYPES.TXT ||
@@ -29,7 +27,7 @@ module.exports = {
         return JSON.parse(parsed);
       } catch (e) {
         console.warn(
-          "BuiltWith sent an invalid JSON payload. Falling back to text parsing."
+          "BuiltWith sent an invalid JSON payload. Falling back to text parsing.",
         );
         return parsed;
       }
@@ -41,7 +39,7 @@ module.exports = {
    * { paramOne: 'hello', paramTwo: 'goodbye' }
    * @param {Object} params
    */
-  paramsObjToQueryString: function(params) {
+  paramsObjToQueryString: function (params) {
     // '&paramOne=hello&paramTwo=goodbye
     return Object.entries(params) // [['paramOne', 'hello'], ['paramTwo', 'goodbye]]
       .filter(([, value]) => {
@@ -52,5 +50,5 @@ module.exports = {
         return `${key}=${value}`; // ['paramOne=hello', 'paramTwo=goodbye']
       })
       .join("&"); // 'paramOne=hello&paramTwo=goodBye'
-  }
-}
+  },
+};