devtools-tracing 1.0.0

package/lib/front_end/third_party/third-party-web/package/lib/create-entity-finder-api.js

@@ -0,0 +1,139 @@
+const DOMAIN_IN_URL_REGEX = /:\/\/(\S*?)(:\d+)?(\/|$)/
+const DOMAIN_CHARACTERS = /([a-z0-9.-]+\.[a-z0-9]+|localhost)/i
+const IP_REGEX = /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/
+const ROOT_DOMAIN_REGEX = /[^.]+\.([^.]+|(gov|com|co|ne)\.\w{2})$/i
+
+function getDomainFromOriginOrURL(originOrURL) {
+  if (typeof originOrURL !== 'string') return null
+  if (originOrURL.length > 10000 || originOrURL.startsWith('data:')) return null
+  if (DOMAIN_IN_URL_REGEX.test(originOrURL)) return originOrURL.match(DOMAIN_IN_URL_REGEX)[1]
+  if (DOMAIN_CHARACTERS.test(originOrURL)) return originOrURL.match(DOMAIN_CHARACTERS)[0]
+  return null
+}
+
+function getRootDomain(originOrURL) {
+  const domain = getDomainFromOriginOrURL(originOrURL)
+  if (!domain) return null
+  if (IP_REGEX.test(domain)) return domain
+  const match = domain.match(ROOT_DOMAIN_REGEX)
+  return (match && match[0]) || domain
+}
+
+function sliceSubdomainFromDomain(domain, rootDomain) {
+  if (domain.length <= rootDomain.length) return domain
+  return domain
+    .split('.')
+    .slice(1)
+    .join('.')
+}
+
+function getEntityInDataset(entityByDomain, entityBySubDomain, entityByRootDomain, originOrURL) {
+  const domain = getDomainFromOriginOrURL(originOrURL)
+  const rootDomain = getRootDomain(domain)
+  if (!domain || !rootDomain) return undefined
+  if (entityByDomain.has(domain)) return entityByDomain.get(domain)
+
+  for (
+    let subdomain = domain;
+    subdomain.length > rootDomain.length;
+    subdomain = sliceSubdomainFromDomain(subdomain, rootDomain)
+  ) {
+    if (entityBySubDomain.has(subdomain)) return entityBySubDomain.get(subdomain)
+  }
+
+  if (entityByRootDomain.has(rootDomain)) return entityByRootDomain.get(rootDomain)
+  return undefined
+}
+
+function getProductInDataset(entityByDomain, entityBySubDomain, entityByRootDomain, originOrURL) {
+  const entity = getEntityInDataset(
+    entityByDomain,
+    entityBySubDomain,
+    entityByRootDomain,
+    originOrURL
+  )
+  const products = entity && entity.products
+  if (!products) return undefined
+  if (typeof originOrURL !== 'string') return undefined
+
+  for (const product of products) {
+    for (const pattern of product.urlPatterns) {
+      if (pattern instanceof RegExp && pattern.test(originOrURL)) return product
+      if (typeof pattern === 'string' && originOrURL.includes(pattern)) return product
+    }
+  }
+  return undefined
+}
+
+function cloneEntities(entities) {
+  return entities.map(entity_ => {
+    const entity = {
+      company: entity_.name,
+      categories: [entity_.category],
+      ...entity_,
+    }
+
+    const products = (entity_.products || []).map(product => ({
+      company: entity.company,
+      category: entity.category,
+      categories: [entity.category],
+      facades: [],
+      ...product,
+      urlPatterns: (product.urlPatterns || []).map(s =>
+        s.startsWith('REGEXP:') ? new RegExp(s.slice('REGEXP:'.length)) : s
+      ),
+    }))
+
+    entity.products = products
+    return entity
+  })
+}
+
+function createAPIFromDataset(entities_) {
+  const entities = cloneEntities(entities_)
+  const entityByDomain = new Map()
+  const entityByRootDomain = new Map()
+  const entityBySubDomain = new Map()
+
+  for (const entity of entities) {
+    entity.totalExecutionTime = Number(entity.totalExecutionTime) || 0
+    entity.totalOccurrences = Number(entity.totalOccurrences) || 0
+    entity.averageExecutionTime = entity.totalExecutionTime / entity.totalOccurrences
+
+    for (const domain of entity.domains) {
+      if (entityByDomain.has(domain)) {
+        const duplicate = entityByDomain.get(domain)
+        throw new Error(`Duplicate domain ${domain} (${entity.name} and ${duplicate.name})`)
+      }
+
+      entityByDomain.set(domain, entity)
+
+      const rootDomain = getRootDomain(domain)
+      if (domain.startsWith('*.')) {
+        const wildcardDomain = domain.slice(2)
+        if (wildcardDomain === rootDomain) entityByRootDomain.set(rootDomain, entity)
+        else entityBySubDomain.set(wildcardDomain, entity)
+      }
+    }
+  }
+
+  for (const [rootDomain, entity] of entityByRootDomain.entries()) {
+    if (!entity) entityByRootDomain.delete(rootDomain)
+  }
+
+  const getEntity = getEntityInDataset.bind(
+    null,
+    entityByDomain,
+    entityBySubDomain,
+    entityByRootDomain
+  )
+  const getProduct = getProductInDataset.bind(
+    null,
+    entityByDomain,
+    entityBySubDomain,
+    entityByRootDomain
+  )
+  return {getEntity, getProduct, getRootDomain, entities}
+}
+
+module.exports = {createAPIFromDataset}

package/lib/front_end/third_party/third-party-web/package/lib/create-entity-finder-api.test.js

@@ -0,0 +1,44 @@
+const {createAPIFromDataset} = require('./create-entity-finder-api.js')
+
+describe('getEntity', () => {
+  let api
+
+  beforeEach(() => {
+    api = createAPIFromDataset([
+      {
+        name: 'Domain',
+        domains: ['*.example.com', '*.example.co.uk'],
+      },
+      {
+        name: 'Subdomain',
+        domains: ['*.sub.example.com', '*.sub.example.co.uk'],
+      },
+      {
+        name: 'Subsubdomain',
+        domains: ['very.specific.example.com'],
+      },
+    ])
+  })
+
+  it('should find direct domains', () => {
+    expect(api.getEntity('https://very.specific.example.com/path').name).toEqual('Subsubdomain')
+  })
+
+  it('should find wildcard subdomains', () => {
+    expect(api.getEntity('https://foo.sub.example.com/path').name).toEqual('Subdomain')
+    expect(api.getEntity('https://bar.sub.example.com/path').name).toEqual('Subdomain')
+    expect(api.getEntity('https://baz.bar.sub.example.com/path').name).toEqual('Subdomain')
+    expect(api.getEntity('https://foo.sub.example.co.uk/path').name).toEqual('Subdomain')
+    expect(api.getEntity('https://bar.sub.example.co.uk/path').name).toEqual('Subdomain')
+    expect(api.getEntity('https://baz.bar.sub.example.co.uk/path').name).toEqual('Subdomain')
+  })
+
+  it('should find wildcard domains', () => {
+    expect(api.getEntity('https://foo.example.com/path').name).toEqual('Domain')
+    expect(api.getEntity('https://bar.example.com/path').name).toEqual('Domain')
+    expect(api.getEntity('https://baz.bar.example.com/path').name).toEqual('Domain')
+    expect(api.getEntity('https://foo.example.co.uk/path').name).toEqual('Domain')
+    expect(api.getEntity('https://bar.example.co.uk/path').name).toEqual('Domain')
+    expect(api.getEntity('https://baz.bar.example.co.uk/path').name).toEqual('Domain')
+  })
+})

package/lib/front_end/third_party/third-party-web/package/lib/entities.test.js

@@ -0,0 +1,27 @@
+const _ = require('lodash')
+const {entities, getRootDomain, getEntity} = require('./index.js')
+
+describe('Entities', () => {
+  it('should not have duplicate names', () => {
+    for (let i = 0; i < entities.length; i++) {
+      for (let j = i + 1; j < entities.length; j++) {
+        const nameA = entities[i].name.replace(/\s+/, '').toLowerCase()
+        const nameB = entities[j].name.replace(/\s+/, '').toLowerCase()
+        if (nameA !== nameB) continue
+
+        expect(entities[i]).toBe(entities[j])
+      }
+    }
+  })
+
+  it('should not have non-supported wilcards', () => {
+    for (const entity of entities) {
+      for (const domain of entity.domains) {
+        // Wildcards must be like `*.blah.com`
+        // Wildcards cannot be `*blah.com` or `blah*.com`
+        expect(domain).toEqual(expect.not.stringMatching(/\w\*/))
+        expect(domain).toEqual(expect.not.stringMatching(/\*\w/))
+      }
+    }
+  })
+})

package/lib/front_end/third_party/third-party-web/package/lib/index.d.ts

@@ -0,0 +1,34 @@
+export interface IFacade {
+  name: string
+  repo: string
+}
+
+export interface IProduct {
+  name: string
+  company: string
+  homepage?: string
+  category: string
+  /** @deprecated - Use `category` instead. */
+  categories: string[]
+  urlPatterns?: string[]
+  facades?: IFacade[]
+}
+
+export interface IEntity {
+  name: string
+  company: string
+  homepage?: string
+  category: string
+  /** @deprecated - Use `category` instead. */
+  categories: string[]
+  domains: string[]
+  products?: IProduct[]
+  averageExecutionTime: number
+  totalExecutionTime: number
+  totalOccurrences: number
+}
+
+export declare const entities: IEntity[]
+export declare function getRootDomain(url: string): string
+export declare function getEntity(url: string): IEntity | undefined
+export declare function getProduct(url: string): IProduct | undefined

package/lib/front_end/third_party/third-party-web/package/lib/index.js

@@ -0,0 +1,3 @@
+const {createAPIFromDataset} = require('./create-entity-finder-api.js')
+const entities = require('../dist/entities.json')
+module.exports = createAPIFromDataset(entities)

package/lib/front_end/third_party/third-party-web/package/lib/index.test.js

@@ -0,0 +1,246 @@
+const fs = require('fs')
+const path = require('path')
+const {entities, getRootDomain, getEntity, getProduct} = require('./index.js')
+
+describe('getRootDomain', () => {
+  it('works for IP addresses', () => {
+    expect(getRootDomain('8.8.8.8')).toEqual('8.8.8.8')
+    expect(getRootDomain('192.168.0.1')).toEqual('192.168.0.1')
+  })
+
+  it('works for basic domains', () => {
+    expect(getRootDomain('cdn.cnn.com')).toEqual('cnn.com')
+    expect(getRootDomain('www.hulce.photography')).toEqual('hulce.photography')
+    expect(getRootDomain('api.supercool.io')).toEqual('supercool.io')
+  })
+
+  it('works for country-tlds', () => {
+    expect(getRootDomain('content.yahoo.co.jp')).toEqual('yahoo.co.jp')
+    expect(getRootDomain('go.visit.gov.in')).toEqual('visit.gov.in')
+  })
+
+  it('works for URLs', () => {
+    expect(getRootDomain('https://content.yahoo.co.jp/path/?query=param')).toEqual('yahoo.co.jp')
+    expect(getRootDomain('https://a.b.c.it/path/?query=param&two=2')).toEqual('c.it')
+    expect(getRootDomain('https://foo.bar:433/path/?query=param&two=2')).toEqual('foo.bar')
+  })
+
+  it('works for localhost', () => {
+    expect(getRootDomain('https://localhost:8080/path/?query=param')).toEqual('localhost')
+    expect(getRootDomain('https://localhost/path/?query=param&two=2')).toEqual('localhost')
+    expect(getRootDomain('localhost:9000/path/?query=param&two=2')).toEqual('localhost')
+    expect(getRootDomain('localhost:1200')).toEqual('localhost')
+  })
+
+  it('works for wildcard domains', () => {
+    expect(getRootDomain('*.google.com')).toEqual('google.com')
+    expect(getRootDomain('*.yahoo.co.jp')).toEqual('yahoo.co.jp')
+    expect(getRootDomain('*.hulce.photography')).toEqual('hulce.photography')
+  })
+
+  it('runs on *massive* inputs', () => {
+    const massiveInput = '123456789'.repeat(100e3)
+    expect(getRootDomain(massiveInput)).toEqual(null)
+  })
+
+  it('runs on data URIs', () => {
+    const dataUri = 'data:image/gif;base64,R0lGODlhAQABAIAAAP8AADAAACwAAAAAAQABAAACAkQBADs='
+    expect(getRootDomain(dataUri)).toEqual(null)
+  })
+
+  it('returns null on invalid inputs', () => {
+    expect(getRootDomain('this is not a domain')).toEqual(null)
+    expect(getRootDomain('neither-is-this')).toEqual(null)
+    expect(getRootDomain('http://nor this')).toEqual(null)
+  })
+})
+
+describe('getEntity', () => {
+  it('works for direct domain usage', () => {
+    expect(getEntity('https://js.connect.facebook.net/lib.js')).toMatchInlineSnapshot(`
+      Object {
+        "averageExecutionTime": 347.4278160199557,
+        "categories": Array [
+          "social",
+        ],
+        "category": "social",
+        "company": "Facebook",
+        "domains": Array [
+          "*.facebook.com",
+          "*.atlassbx.com",
+          "*.fbsbx.com",
+          "fbcdn-photos-e-a.akamaihd.net",
+          "*.facebook.net",
+          "*.fbcdn.net",
+        ],
+        "examples": Array [
+          "www.facebook.com",
+          "connect.facebook.net",
+          "staticxx.facebook.com",
+          "static.xx.fbcdn.net",
+          "m.facebook.com",
+          "an.facebook.com",
+          "platform-lookaside.fbsbx.com",
+        ],
+        "homepage": "https://www.facebook.com",
+        "name": "Facebook",
+        "products": Array [
+          Object {
+            "categories": Array [
+              "social",
+            ],
+            "category": "social",
+            "company": "Facebook",
+            "facades": Array [
+              Object {
+                "name": "React Live Chat Loader",
+                "repo": "https://github.com/calibreapp/react-live-chat-loader",
+              },
+            ],
+            "name": "Facebook Messenger Customer Chat",
+            "urlPatterns": Array [
+              /connect\\\\\\.facebook\\\\\\.net\\\\/\\.\\*\\\\/sdk\\\\/xfbml\\\\\\.customerchat\\\\\\.js/,
+            ],
+          },
+        ],
+        "totalExecutionTime": 1097107210,
+        "totalOccurrences": 3157799,
+      }
+    `)
+  })
+
+  it('works for inferred domain usage', () => {
+    expect(getEntity('https://unknown.typekit.net/fonts.css')).toMatchInlineSnapshot(`
+      Object {
+        "averageExecutionTime": 660.2645605704683,
+        "categories": Array [
+          "cdn",
+        ],
+        "category": "cdn",
+        "company": "Adobe",
+        "domains": Array [
+          "*.typekit.com",
+          "*.typekit.net",
+        ],
+        "examples": Array [
+          "use.typekit.net",
+          "p.typekit.net",
+        ],
+        "homepage": "https://fonts.adobe.com/",
+        "name": "Adobe TypeKit",
+        "products": Array [],
+        "totalExecutionTime": 78981507,
+        "totalOccurrences": 119621,
+      }
+    `)
+  })
+
+  it('does not over-infer', () => {
+    expect(getEntity('https://unknown.gstatic.com/what')).toEqual(undefined)
+  })
+
+  it('only infers as a fallback', () => {
+    expect(getEntity('http://fbcdn-photos-e-a.akamaihd.net/1234.jpg').name).toEqual('Facebook')
+    expect(getEntity('http://unknown.akamaihd.net/1234.jpg').name).toEqual('Akamai')
+  })
+
+  it('runs on *massive* inputs', () => {
+    const massiveInput = '123456789'.repeat(100e3)
+    expect(getEntity(massiveInput)).toEqual(undefined)
+  })
+
+  it('runs on data URIs', () => {
+    const dataUri = 'data:image/gif;base64,R0lGODlhAQABAIAAAP8AADAAACwAAAAAAQABAAACAkQBADs='
+    expect(getEntity(dataUri)).toEqual(undefined)
+  })
+
+  it('supports multi-tennant domains', () => {
+    expect(getEntity('https://gemius.mgr.consensu.org/cmp/v2/stub.js').name).toEqual('Gemius CMP')
+    expect(
+      getEntity('https://quantcast.mgr.consensu.org/choice/KygWsHah2_7Qa/rssing.com/choice.js').name
+    ).toEqual('Quantcast Choice')
+    expect(getEntity('https://static.quantcast.mgr.consensu.org/v50/cmpui-popup.js').name).toEqual(
+      'Quantcast Choice'
+    )
+  })
+})
+
+describe('getProduct', () => {
+  it('works on basic url', () => {
+    expect(getProduct('https://www.youtube.com/embed/alGcULGtiv8')).toMatchObject({
+      name: 'YouTube Embedded Player',
+      company: 'YouTube',
+      category: 'video',
+      categories: ['video'],
+      facades: [
+        {
+          name: 'Lite YouTube',
+          repo: 'https://github.com/paulirish/lite-youtube-embed',
+        },
+        {
+          name: 'Ngx Lite Video',
+          repo: 'https://github.com/karim-mamdouh/ngx-lite-video',
+        },
+      ],
+    })
+  })
+
+  it('works on regex based', () => {
+    expect(
+      getProduct('https://connect.facebook.net/en_US/sdk/xfbml.customerchat.js')
+    ).toMatchObject({
+      name: 'Facebook Messenger Customer Chat',
+      facades: [
+        {
+          name: 'React Live Chat Loader',
+          repo: 'https://github.com/calibreapp/react-live-chat-loader',
+        },
+      ],
+    })
+  })
+
+  it('returns undefined when product does not match', () => {
+    expect(getProduct('https://js.connect.facebook.net/lib.js')).toEqual(undefined)
+  })
+
+  it('returns undefined with no products', () => {
+    expect(getProduct('https://unknown.typekit.net/fonts.css')).toEqual(undefined)
+  })
+})
+
+describe('build state', () => {
+  it('should use the complete entities set', () => {
+    const sourceOfTruthEntities = require('../data/entities.js')
+    expect(entities).toHaveLength(sourceOfTruthEntities.length)
+  })
+
+  it('should have all the same subsets in root as lib', () => {
+    const srcSizes = fs.readdirSync(path.join(__dirname, 'subsets'))
+    const dstSizes = fs.readdirSync(path.join(__dirname, '../')).filter(f => f.includes('-subset'))
+    expect(dstSizes).toHaveLength(srcSizes.length) // run `yarn build` if this fails
+
+    for (const file of dstSizes) {
+      if (file.endsWith('.js')) require(path.join(__dirname, '../', file))
+    }
+  })
+})
+
+it('should work on real web data', () => {
+  const urls = fs
+    .readFileSync(path.join(__dirname, '../data/random-urls.txt'), 'utf8')
+    .split('\n')
+    .filter(Boolean)
+
+  for (const url of urls) {
+    getEntity(url) // ensure it doesn't throw
+  }
+
+  const top1000 = urls.slice(0, 1000).map(url => {
+    const cleanedUrl = url.split('?')[0]
+    const entity = getEntity(url)
+    return `${entity && entity.name} - ${cleanedUrl}`
+  })
+
+  // It's expected that this snapshot will change as coverage changes.
+  expect(top1000).toMatchSnapshot()
+})

package/lib/front_end/third_party/third-party-web/package/lib/markdown/faqs.partial.md

@@ -0,0 +1,36 @@
+---
+name: faqs
+---
+
+### I don't see entity X in the list. What's up with that?
+
+This can be for one of several reasons:
+
+1.  The entity does not have references to their origin on at least 50 pages in the dataset.
+1.  The entity's origins have not yet been identified. See [How can I contribute?](#contribute)
+
+### What is "Total Occurences"?
+
+Total Occurrences is the number of pages on which the entity is included.
+
+### How is the "Average Impact" determined?
+
+The HTTP Archive dataset includes Lighthouse reports for each URL on mobile. Lighthouse has an audit called "bootup-time" that summarizes the amount of time that each script spent on the main thread. The "Average Impact" for an entity is the total execution time of scripts whose domain matches one of the entity's domains divided by the total number of pages that included the entity.
+
+```
+Average Impact = Total Execution Time / Total Occurrences
+```
+
+### How does Lighthouse determine the execution time of each script?
+
+Lighthouse's bootup time audit attempts to attribute all toplevel main-thread tasks to a URL. A main thread task is attributed to the first script URL found in the stack. If you're interested in helping us improve this logic, see [Contributing](#contributing) for details.
+
+### The data for entity X seems wrong. How can it be corrected?
+
+Verify that the origins in `data/entities.js` are correct. Most issues will simply be the result of mislabelling of shared origins. If everything checks out, there is likely no further action and the data is valid. If you still believe there's errors, file an issue to discuss futher.
+
+<a name="contribute"></a>
+
+### How can I contribute?
+
+Only about 90% of the third party script execution has been assigned to an entity. We could use your help identifying the rest! See [Contributing](#contributing) for details.

package/lib/front_end/third_party/third-party-web/package/lib/markdown/goals.partial.md

@@ -0,0 +1,9 @@
+---
+name: goals
+---
+
+1.  Quantify the impact of third party scripts on the web.
+1.  Identify the third party scripts on the web that have the greatest performance cost.
+1.  Give developers the information they need to make informed decisions about which third parties to include on their sites.
+1.  Incentivize responsible third party script behavior.
+1.  Make this information accessible and useful.

package/lib/front_end/third_party/third-party-web/package/lib/markdown/methodology.partial.md

@@ -0,0 +1,5 @@
+---
+name: methodology
+---
+
+[HTTP Archive](https://httparchive.org/) is an initiative that tracks how the web is built. Every month, ~4 million sites are crawled with [Lighthouse](https://github.com/GoogleChrome/lighthouse) on mobile. Lighthouse breaks down the total script execution time of each page and attributes the execution to a URL. Using [BigQuery](https://cloud.google.com/bigquery/), this project aggregates the script execution to the origin-level and assigns each origin to the responsible entity.