devtools-tracing 1.0.0

package/lib/front_end/third_party/third-party-web/package/lib/markdown/template.md

@@ -0,0 +1,151 @@
+# [Third Party Web](https://www.thirdpartyweb.today/)
+
+## Check out the shiny new web UI https://www.thirdpartyweb.today/
+
+Data on third party entities and their impact on the web.
+
+This document is a summary of which third party scripts are most responsible for excessive JavaScript execution on the web today.
+
+## Table of Contents
+
+1.  [Goals](#goals)
+1.  [Methodology](#methodology)
+1.  [npm Module](#npm-module)
+1.  [Updates](#updates)
+1.  [Data](#data)
+    1.  [Summary](#summary)
+    1.  [How to Interpret](#how-to-interpret)
+    1.  [Third Parties by Category](#by-category)
+        <%= category_table_of_contents %>
+    1.  [Third Parties by Total Impact](#by-total-impact)
+1.  [Future Work](#future-work)
+1.  [FAQs](#faqs)
+1.  [Contributing](#contributing)
+
+## Goals
+
+<%= partials.goals %>
+
+## Methodology
+
+<%= partials.methodology %>
+
+## npm Module
+
+The entity classification data is available as an npm module.
+
+```js
+const {getEntity} = require('third-party-web')
+const entity = getEntity('https://d36mpcpuzc4ztk.cloudfront.net/js/visitor.js')
+console.log(entity)
+//   {
+//     "name": "Freshdesk",
+//     "homepage": "https://freshdesk.com/",
+//     "category": "customer-success",
+//     "domains": ["d36mpcpuzc4ztk.cloudfront.net"]
+//   }
+```
+
+## Updates
+
+<%= updates_contents %>
+
+## Data
+
+### Summary
+
+Across top ~4 million sites, ~2700 origins account for ~57% of all script execution time with the top 50 entities already accounting for ~47%. Third party script execution is the majority chunk of the web today, and it's important to make informed choices.
+
+### How to Interpret
+
+Each entity has a number of data points available.
+
+1.  **Usage (Total Number of Occurrences)** - how many scripts from their origins were included on pages
+1.  **Total Impact (Total Execution Time)** - how many seconds were spent executing their scripts across the web
+1.  **Average Impact (Average Execution Time)** - on average, how many milliseconds were spent executing each script
+1.  **Category** - what type of script is this
+
+<a name="by-category"></a>
+
+### Third Parties by Category
+
+This section breaks down third parties by category. The third parties in each category are ranked from first to last based on the average impact of their scripts. Perhaps the most important comparisons lie here. You always need to pick an analytics provider, but at least you can pick the most well-behaved analytics provider.
+
+#### Overall Breakdown
+
+Unsurprisingly, ads account for the largest identifiable chunk of third party script execution.
+
+![breakdown by category](./by-category.png)
+
+<%= category_contents %>
+
+<a name="by-total-impact"></a>
+
+### Third Parties by Total Impact
+
+This section highlights the entities responsible for the most script execution across the web. This helps inform which improvements would have the largest total impact.
+
+<%= all_data %>
+
+## Future Work
+
+1.  Introduce URL-level data for more fine-grained analysis, i.e. which libraries from Cloudflare/Google CDNs are most expensive.
+1.  Expand the scope, i.e. include more third parties and have greater entity/category coverage.
+
+## FAQs
+
+<%= partials.faqs %>
+
+## Contributing
+
+### Thanks
+
+A **huge** thanks to [@simonhearne](https://twitter.com/simonhearne) and [@soulgalore](https://twitter.com/soulislove) for their assistance in classifying additional domains!
+
+### Updating the Entities
+
+The domain->entity mapping can be found in `data/entities.js`. Adding a new entity is as simple as adding a new array item with the following form.
+
+```js
+{
+    "name": "Facebook",
+    "homepage": "https://www.facebook.com",
+    "category": "social",
+    "domains": [
+        "*.facebook.com",
+        "*.fbcdn.net"
+    ],
+    "examples": [
+        "www.facebook.com",
+        "connect.facebook.net",
+        "staticxx.facebook.com",
+        "static.xx.fbcdn.net",
+        "m.facebook.com"
+    ]
+}
+```
+
+### Updating Attribution Logic
+
+The logic for attribution to individual script URLs can be found in the [Lighthouse repo](https://github.com/GoogleChrome/lighthouse). File an issue over there to discuss further.
+
+### Updating the Data
+
+This is now automated! Run `yarn start:update-ha-data` with a `gcp-credentials.json` file in the root directory of this project (look at `bin/automated-update.js` for the steps involved).
+
+### Updating this README
+
+This README is auto-generated from the templates `lib/` and the computed data. In order to update the charts, you'll need to make sure you have `cairo` installed locally in addition to `yarn install`.
+
+```bash
+# Install `cairo` and dependencies for node-canvas
+brew install pkg-config cairo pango libpng jpeg giflib
+# Build the requirements in this repo
+yarn build
+# Regenerate the README
+yarn start
+```
+
+### Updating the website
+
+The web code is located in `www/` directory of this repository. Open a PR to make changes.

package/lib/front_end/third_party/third-party-web/package/lib/markdown/updates/2019-02-01.md

@@ -0,0 +1 @@
+Huge props to [WordAds](https://wordads.co/) for reducing their impact from ~2.5s to ~200ms on average! A few entities are showing considerably less data this cycle (Media Math, Crazy Egg, DoubleVerify, Bootstrap CDN). Perhaps they've added new CDNs/hostnames that we haven't identified or the basket of sites in HTTPArchive has shifted away from their usage.

package/lib/front_end/third_party/third-party-web/package/lib/markdown/updates/2019-03-01.md

@@ -0,0 +1 @@
+Almost 2,000 entities tracked now across ~3,000+ domains! Huge props to [@simonhearne](https://twitter.com/simonhearne) for making this massive increase possible. Tag Managers have now been split out into their own category since they represented such a large percentage of the "Mixed / Other" category.

package/lib/front_end/third_party/third-party-web/package/lib/markdown/updates/2019-05-06.md

@@ -0,0 +1 @@
+Google Ads clarified that `www.googletagservices.com` serves more ad scripts than generic tag management, and it has been reclassified accordingly. This has dropped the overall Tag Management share considerably back down to its earlier position.

package/lib/front_end/third_party/third-party-web/package/lib/markdown/updates/2019-05-13.md

@@ -0,0 +1,14 @@
+A shortcoming of the attribution approach has been fixed. Total usage is now reported based on the number of _pages_ in the dataset that use the third-party, not the number of _scripts_. Correspondingly, all average impact times are now reported _per page_ rather than _per script_. Previously, a third party could appear to have a lower impact or be more popular simply by splitting their work across multiple files.
+
+Third-parties that performed most of their work from a single script should see little to no impact from this change, but some entities have seen significant ranking movement. Hosting providers that host entire pages are, understandably, the most affected.
+
+Some notable changes below:
+
+| Third-Party | Previously (per-script) | Now (per-page) |
+| ----------- | ----------------------- | -------------- |
+| Beeketing   | 137 ms                  | 465 ms         |
+| Sumo        | 263 ms                  | 798 ms         |
+| Tumblr      | 324 ms                  | 1499 ms        |
+| Yandex APIs | 393 ms                  | 1231 ms        |
+| Google Ads  | 402 ms                  | 1285 ms        |
+| Wix         | 972 ms                  | 5393 ms        |

package/lib/front_end/third_party/third-party-web/package/lib/markdown/updates/2021-01-01.md

@@ -0,0 +1 @@
+Due to a change in HTTPArchive measurement which temporarily disabled site-isolation (out-of-process iframes), all of the third-parties whose work previously took place off the main-thread are now counted _on_ the main thread (and thus appear in our stats). This is most evident in the change to Google-owned properties such as YouTube and Doubleclick whose _complete_ cost are now captured.

package/lib/front_end/third_party/third-party-web/package/lib/markdown/updates/2024-07-01.md

@@ -0,0 +1,3 @@
+Some third parties use a dynamic subdomain to serve its main script on websites (e.g .domain.com). Some of these subdomain scripts are saved under observed-domains JSON file as results of the `sql/all-observed-domains-query.sql` query but analyzing http archive database we found a lot that are ignored because of number of occurrences (less than 50 ).
+
+So, we've created a new query to keep observed domains with occurrence below 50 only if its mapped entity (based on entity.js) has a total occurrence (of all its declared domain) greater than 50.

package/lib/front_end/third_party/third-party-web/package/lib/subsets/httparchive-nostats.d.ts

@@ -0,0 +1 @@
+export * from '../index'

package/lib/front_end/third_party/third-party-web/package/lib/subsets/httparchive-nostats.js

@@ -0,0 +1,3 @@
+const {createAPIFromDataset} = require('../create-entity-finder-api.js')
+const entities = require('../../dist/entities-httparchive-nostats.json')
+module.exports = createAPIFromDataset(entities)

package/lib/front_end/third_party/third-party-web/package/lib/subsets/httparchive.d.ts

@@ -0,0 +1 @@
+export * from '../index'

package/lib/front_end/third_party/third-party-web/package/lib/subsets/httparchive.js

@@ -0,0 +1,3 @@
+const {createAPIFromDataset} = require('../create-entity-finder-api.js')
+const entities = require('../../dist/entities-httparchive.json')
+module.exports = createAPIFromDataset(entities)

package/lib/front_end/third_party/third-party-web/package/lib/subsets/nostats.d.ts

@@ -0,0 +1 @@
+export * from '../index'

package/lib/front_end/third_party/third-party-web/package/lib/subsets/nostats.js

@@ -0,0 +1,3 @@
+const {createAPIFromDataset} = require('../create-entity-finder-api.js')
+const entities = require('../../dist/entities-nostats.json')
+module.exports = createAPIFromDataset(entities)

package/lib/front_end/third_party/third-party-web/package/nostats-subset.d.ts

@@ -0,0 +1 @@
+export * from './lib/subsets/nostats.js'

package/lib/front_end/third_party/third-party-web/package/nostats-subset.js

@@ -0,0 +1 @@
+module.exports = require("./lib/subsets/nostats.js")

package/lib/front_end/third_party/third-party-web/package/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "third-party-web",
+  "version": "0.26.2",
+  "description": "Categorized data on third party entities on the web.",
+  "main": "./lib/index.js",
+  "scripts": {
+    "build:www": "cd www/ && npm install && npm run build",
+    "build": "node bin/convert-entities-db.js && node bin/generate-canonical-domain-csv.js && node bin/build-entity-json-files.js && node bin/generate-sql.js && generate-export-aliases",
+    "lint:fix": "prettier --print-width=100 --write '**/*.js'",
+    "start": "mkdir -p .tmp && node bin/merge-origins-with-entities.js && node bin/create-markdown.js",
+    "start:update-ha-data": "node ./bin/automated-update.js",
+    "test": "yarn build && yarn test:unit",
+    "test:unit": "jest 'lib/*'",
+    "semantic-release": "semantic-release"
+  },
+  "keywords": [
+    "third-party",
+    "third-parties"
+  ],
+  "author": "Patrick Hulce <patrick.hulce@gmail.com>",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/patrickhulce/third-party-web.git"
+  },
+  "license": "MIT",
+  "devDependencies": {
+    "@google-cloud/bigquery": "^7.9.0",
+    "chart.js": "^2.9.4",
+    "chartjs-node-canvas": "^3.2.0",
+    "colors": "^1.4.0",
+    "generate-export-aliases": "^1.1.0",
+    "jest": "^24.9.0",
+    "lodash": "^4.17.15",
+    "prettier": "^1.18.2",
+    "prompts": "^2.4.2",
+    "semantic-release": "^24.2.0"
+  },
+  "config": {
+    "exportAliases": {
+      "nostats-subset": "./lib/subsets/nostats.js",
+      "httparchive-nostats-subset": "./lib/subsets/httparchive-nostats.js",
+      "httparchive-subset": "./lib/subsets/httparchive.js"
+    }
+  },
+  "packageManager": "yarn@4.1.1"
+}

package/lib/front_end/third_party/third-party-web/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "third-party-web",
+  "description": "Package list used when building/upgrading third-party-web",
+  "private": true,
+  "dependencies": {
+    "third-party-web": "0.26.2"
+  }
+}

package/lib/front_end/third_party/third-party-web/rebuild.sh

@@ -0,0 +1,13 @@
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+cd "$SCRIPT_DIR"
+
+npm install
+
+rm -rf package
+cp -r node_modules/third-party-web package
+rm -fr package/.* package/www-v2 package/dist/domain-map.csv
+
+../../../third_party/esbuild/esbuild  --bundle package/nostats-subset.js --format=esm --outdir=lib
+rm -rf node_modules
+# Because there's a bug in clang causing it to reformat import lists even where formatting is disabled, run it right away
+git cl format --js

package/lib/front_end/third_party/third-party-web/third-party-web-tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "compilerOptions": {
+    "composite": true
+  },
+  "files": [
+    "lib/nostats-subset.js"
+  ]
+}

package/lib/front_end/third_party/third-party-web/third-party-web.ts

@@ -0,0 +1,3 @@
+import ThirdPartyWeb from './lib/nostats-subset.js';
+
+export {ThirdPartyWeb};

package/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "devtools-tracing",
+  "version": "1.0.0",
+  "description": "Utilities for working with trace files",
+  "main": "index.js",
+  "scripts": {
+    "start": "tsx index.ts",
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "postinstall": "patch-package"
+  },
+  "author": "",
+  "license": "ISC",
+  "devDependencies": {
+    "@typescript-eslint/typescript-estree": "^8.46.2",
+    "chrome-devtools-frontend": "^1.0.1533544",
+    "patch-package": "^8.0.1",
+    "tsx": "^4.20.6",
+    "typescript": "^5.9.3"
+  },
+  "prettier": {
+    "tabWidth": 2,
+    "singleQuote": true
+  }
+}

package/patches/chrome-devtools-frontend+1.0.1533544.patch

@@ -0,0 +1,187 @@
+diff --git a/node_modules/chrome-devtools-frontend/front_end/models/trace/handlers/helpers.ts b/node_modules/chrome-devtools-frontend/front_end/models/trace/handlers/helpers.ts
+index 62c5773..ff07f89 100644
+--- a/node_modules/chrome-devtools-frontend/front_end/models/trace/handlers/helpers.ts
++++ b/node_modules/chrome-devtools-frontend/front_end/models/trace/handlers/helpers.ts
+@@ -3,13 +3,13 @@
+ // found in the LICENSE file.
+ 
+ import type * as Platform from '../../../core/platform/platform.js';
+-import * as ThirdPartyWeb from '../../../third_party/third-party-web/third-party-web.js';
+ import * as Types from '../types/types.js';
+ 
+ import type {TraceEventsForNetworkRequest} from './NetworkRequestsHandler.js';
+ import type {HandlerData} from './types.js';
+ 
+-export type Entity = typeof ThirdPartyWeb.ThirdPartyWeb.entities[number]&{
++export type Entity = {
++  name?: string,
+   isUnrecognized?: boolean,
+ };
+ 
+@@ -33,7 +33,7 @@ export function getEntityForUrl(url: string, entityMappings: EntityMappings): En
+   if (cachedByUrl) {
+     return cachedByUrl;
+   }
+-  const entity = ThirdPartyWeb.ThirdPartyWeb.getEntity(url) ?? makeUpEntity(entityMappings.createdEntityCache, url);
++  const entity = makeUpEntity(entityMappings.createdEntityCache, url);
+   if (entity) {
+     entityMappings.entityByUrlCache.set(url, entity);
+   }
+@@ -105,17 +105,7 @@ export function makeUpEntity(entityCache: Map<string, Entity>, url: string): Ent
+     return;
+   }
+ 
+-  // NOTE: Lighthouse uses a tld database to determine the root domain, but here
+-  // we are using third party web's database. Doesn't really work for the case of classifying
+-  // domains 3pweb doesn't know about, so it will just give us a guess.
+-  const rootDomain = ThirdPartyWeb.ThirdPartyWeb.getRootDomain(url);
+-  if (!rootDomain) {
+-    return;
+-  }
+-
+-  if (entityCache.has(rootDomain)) {
+-    return entityCache.get(rootDomain);
+-  }
++  const rootDomain = URL.parse(url)?.hostname || url
+ 
+   const unrecognizedEntity = {
+     name: rootDomain,
+diff --git a/node_modules/chrome-devtools-frontend/front_end/panels/timeline/TimelineUIUtils.ts b/node_modules/chrome-devtools-frontend/front_end/panels/timeline/TimelineUIUtils.ts
+index 4fe7b92..573e0d3 100644
+--- a/node_modules/chrome-devtools-frontend/front_end/panels/timeline/TimelineUIUtils.ts
++++ b/node_modules/chrome-devtools-frontend/front_end/panels/timeline/TimelineUIUtils.ts
+@@ -48,7 +48,6 @@ import * as Tracing from '../../services/tracing/tracing.js';
+ import * as CodeHighlighter from '../../ui/components/code_highlighter/code_highlighter.js';
+ // eslint-disable-next-line @devtools/es-modules-import
+ import codeHighlighterStyles from '../../ui/components/code_highlighter/codeHighlighter.css.js';
+-import * as PerfUI from '../../ui/legacy/components/perf_ui/perf_ui.js';
+ // eslint-disable-next-line @devtools/es-modules-import
+ import imagePreviewStyles from '../../ui/legacy/components/utils/imagePreview.css.js';
+ import * as LegacyComponents from '../../ui/legacy/components/utils/utils.js';
+@@ -1531,14 +1530,6 @@ export class TimelineUIUtils {
+       TimelineUIUtils.renderEventJson(event, contentHelper);
+     }
+ 
+-    const stats: TimeRangeCategoryStats = {};
+-    const showPieChart = canShowPieChart && TimelineUIUtils.aggregatedStatsForTraceEvent(stats, parsedTrace, event);
+-    if (showPieChart) {
+-      contentHelper.addSection(i18nString(UIStrings.aggregatedTime));
+-      const pieChart = TimelineUIUtils.generatePieChart(stats, TimelineUIUtils.eventStyle(event).category, selfTime);
+-      contentHelper.appendElementRow('', pieChart);
+-    }
+-
+     return contentHelper.fragment;
+   }
+ 
+@@ -2048,79 +2039,6 @@ export class TimelineUIUtils {
+     return Trace.Styles.getCategoryStyles();
+   }
+ 
+-  static generatePieChart(
+-      aggregatedStats: TimeRangeCategoryStats, selfCategory?: Trace.Styles.TimelineCategory,
+-      selfTime?: Trace.Types.Timing.Micro): Element {
+-    let total = 0;
+-    for (const categoryName in aggregatedStats) {
+-      total += aggregatedStats[categoryName];
+-    }
+-
+-    const element = document.createElement('div');
+-    element.classList.add('timeline-details-view-pie-chart-wrapper');
+-    element.classList.add('hbox');
+-
+-    const pieChart = new PerfUI.PieChart.PieChart();
+-    const slices: Array<{
+-      value: number,
+-      color: string,
+-      title: string,
+-    }> = [];
+-
+-    function appendLegendRow(title: string, value: number, color: string): void {
+-      if (!value) {
+-        return;
+-      }
+-      slices.push({value, color, title});
+-    }
+-
+-    // In case of self time, first add self, then children of the same category.
+-    if (selfCategory) {
+-      const selfTimeMilli = Trace.Helpers.Timing.microToMilli(selfTime || 0 as Trace.Types.Timing.Micro);
+-      if (selfTime) {
+-        appendLegendRow(
+-            i18nString(UIStrings.sSelf, {PH1: selfCategory.title}),
+-            selfTimeMilli,
+-            selfCategory.getCSSValue(),
+-        );
+-      }
+-      // Children of the same category.
+-      const categoryTime = aggregatedStats[selfCategory.name];
+-      const value = categoryTime - (selfTimeMilli || 0);
+-      if (value > 0) {
+-        appendLegendRow(
+-            i18nString(UIStrings.sChildren, {PH1: selfCategory.title}),
+-            value,
+-            selfCategory.getCSSValue(),
+-        );
+-      }
+-    }
+-
+-    // Add other categories.
+-    for (const categoryName in Trace.Styles.getCategoryStyles()) {
+-      const category = Trace.Styles.getCategoryStyles()[categoryName as keyof Trace.Styles.CategoryPalette];
+-      if (categoryName === selfCategory?.name) {
+-        // Do not add an entry for this event's self category because 2
+-        // entries for it where added just before this for loop (for
+-        // self and children times).
+-        continue;
+-      }
+-      appendLegendRow(category.title, aggregatedStats[category.name], category.getCSSValue());
+-    }
+-
+-    pieChart.data = {
+-      chartName: i18nString(UIStrings.timeSpentInRendering),
+-      size: 110,
+-      formatter: (value: number) => i18n.TimeUtilities.preciseMillisToString(value),
+-      showLegend: true,
+-      total,
+-      slices,
+-    };
+-    const pieChartContainer = element.createChild('div', 'vbox');
+-    pieChartContainer.appendChild(pieChart);
+-
+-    return element;
+-  }
+   // Generates a Summary component given a aggregated stats for categories.
+   static generateSummaryDetails(
+       aggregatedStats: Record<string, number>, rangeStart: number, rangeEnd: number,
+@@ -2174,31 +2092,6 @@ export class TimelineUIUtils {
+     return element;
+   }
+ 
+-  static generateDetailsContentForFrame(
+-      frame: Trace.Types.Events.LegacyTimelineFrame, filmStrip: Trace.Extras.FilmStrip.Data|null,
+-      filmStripFrame: Trace.Extras.FilmStrip.Frame|null): DocumentFragment {
+-    const contentHelper = new TimelineDetailsContentHelper(null, null);
+-    contentHelper.addSection(i18nString(UIStrings.frame));
+-
+-    const duration = TimelineUIUtils.frameDuration(frame);
+-    contentHelper.appendElementRow(i18nString(UIStrings.duration), duration);
+-    if (filmStrip && filmStripFrame) {
+-      const filmStripPreview = document.createElement('div');
+-      filmStripPreview.classList.add('timeline-filmstrip-preview');
+-      // TODO(paulirish): Adopt Util.ImageCache
+-      const uri = Trace.Handlers.ModelHandlers.Screenshots.screenshotImageDataUri(filmStripFrame.screenshotEvent);
+-      void UI.UIUtils.loadImage(uri).then(image => image && filmStripPreview.appendChild(image));
+-      contentHelper.appendElementRow('', filmStripPreview);
+-      filmStripPreview.addEventListener('click', frameClicked.bind(null, filmStrip, filmStripFrame), false);
+-    }
+-
+-    function frameClicked(filmStrip: Trace.Extras.FilmStrip.Data, filmStripFrame: Trace.Extras.FilmStrip.Frame): void {
+-      PerfUI.FilmStripView.Dialog.fromFilmStrip(filmStrip, filmStripFrame.index);
+-    }
+-
+-    return contentHelper.fragment;
+-  }
+-
+   static frameDuration(frame: Trace.Types.Events.LegacyTimelineFrame): Element {
+     const offsetMilli = Trace.Helpers.Timing.microToMilli(frame.startTimeOffset);
+     const durationMilli = Trace.Helpers.Timing.microToMilli(Trace.Types.Timing.Micro(frame.endTime - frame.startTime));