lighthouse 9.5.0-dev.20221129 → 9.5.0-dev.20221201

package/core/lib/dependency-graph/simulator/simulator-timing-map.js

@@ -39,10 +39,18 @@ import {BaseNode} from '../base-node.js';
 /** @typedef {NetworkNodeTimingStarted & Pick<NodeTimingComplete, 'estimatedTimeElapsed'>} NetworkNodeTimingInProgress */
 
 /** @typedef {CpuNodeTimingInProgress & Pick<NodeTimingComplete, 'endTime'>} CpuNodeTimingComplete */
-/** @typedef {NetworkNodeTimingInProgress & Pick<NodeTimingComplete, 'endTime'>} NetworkNodeTimingComplete */
+/** @typedef {NetworkNodeTimingInProgress & Pick<NodeTimingComplete, 'endTime'> & {connectionTiming: ConnectionTiming}} NetworkNodeTimingComplete */
 
 /** @typedef {NodeTimingQueued | CpuNodeTimingStarted | NetworkNodeTimingStarted | CpuNodeTimingInProgress | NetworkNodeTimingInProgress | CpuNodeTimingComplete | NetworkNodeTimingComplete} NodeTimingData */
 
+/**
+ * @typedef ConnectionTiming A breakdown of network connection timings.
+ * @property {number} [dnsResolutionTime]
+ * @property {number} [connectionTime]
+ * @property {number} [sslTime]
+ * @property {number} timeToFirstByte
+ */
+
 class SimulatorTimingMap {
   constructor() {
     /** @type {Map<Node, NodeTimingData>} */
@@ -83,12 +91,13 @@ class SimulatorTimingMap {
 
   /**
    * @param {Node} node
-   * @param {{endTime: number}} values
+   * @param {{endTime: number, connectionTiming?: ConnectionTiming}} values
    */
   setCompleted(node, values) {
     const nodeTiming = {
       ...this.getInProgress(node),
       endTime: values.endTime,
+      connectionTiming: values.connectionTiming,
     };
 
     this._nodeTimings.set(node, nodeTiming);

package/core/lib/dependency-graph/simulator/simulator.js

@@ -16,6 +16,8 @@ const mobileSlow4G = constants.throttling.mobileSlow4G;
 /** @typedef {import('../base-node.js').Node} Node */
 /** @typedef {import('../network-node').NetworkNode} NetworkNode */
 /** @typedef {import('../cpu-node').CPUNode} CpuNode */
+/** @typedef {import('./simulator-timing-map.js').CpuNodeTimingComplete | import('./simulator-timing-map.js').NetworkNodeTimingComplete} CompleteNodeTiming */
+/** @typedef {import('./simulator-timing-map.js').ConnectionTiming} ConnectionTiming */
 
 // see https://cs.chromium.org/search/?q=kDefaultMaxNumDelayableRequestsPerClient&sq=package:chromium&type=cs
 const DEFAULT_MAXIMUM_CONCURRENT_REQUESTS = 10;
@@ -40,7 +42,7 @@ const PriorityStartTimePenalty = {
   VeryLow: 2,
 };
 
-/** @type {Map<string, LH.Gatherer.Simulation.Result['nodeTimings']>} */
+/** @type {Map<string, Map<Node, CompleteNodeTiming>>} */
 const ALL_SIMULATION_NODE_TIMINGS = new Map();
 
 class Simulator {
@@ -167,12 +169,13 @@ class Simulator {
   /**
    * @param {Node} node
    * @param {number} endTime
+   * @param {ConnectionTiming} [connectionTiming] Optional network connection information.
    */
-  _markNodeAsComplete(node, endTime) {
+  _markNodeAsComplete(node, endTime, connectionTiming) {
     this._nodes[NodeState.Complete].add(node);
     this._nodes[NodeState.InProgress].delete(node);
     this._numberInProgressByType.set(node.type, this._numberInProgress(node.type) - 1);
-    this._nodeTimings.setCompleted(node, {endTime});
+    this._nodeTimings.setCompleted(node, {endTime, connectionTiming});
 
     // Try to add all its dependents to the queue
     for (const dependent of node.getDependents()) {
@@ -371,7 +374,7 @@ class Simulator {
     if (isFinished) {
       connection.setWarmed(true);
       this._connectionPool.release(record);
-      this._markNodeAsComplete(node, totalElapsedTime);
+      this._markNodeAsComplete(node, totalElapsedTime, calculation.connectionTiming);
     } else {
       timingData.timeElapsed += calculation.timeElapsed;
       timingData.timeElapsedOvershoot += calculation.timeElapsed - timePeriodLength;
@@ -380,23 +383,31 @@ class Simulator {
   }
 
   /**
-   * @return {Map<Node, LH.Gatherer.Simulation.NodeTiming>}
+   * @return {{nodeTimings: Map<Node, LH.Gatherer.Simulation.NodeTiming>, completeNodeTimings: Map<Node, CompleteNodeTiming>}}
    */
   _computeFinalNodeTimings() {
+    /** @type {Array<[Node, CompleteNodeTiming]>} */
+    const completeNodeTimingEntries = this._nodeTimings.getNodes().map(node => {
+      return [node, this._nodeTimings.getCompleted(node)];
+    });
+
+    // Most consumers will want the entries sorted by startTime, so insert them in that order
+    completeNodeTimingEntries.sort((a, b) => a[1].startTime - b[1].startTime);
+
+    // Trimmed version of type `LH.Gatherer.Simulation.NodeTiming`.
     /** @type {Array<[Node, LH.Gatherer.Simulation.NodeTiming]>} */
-    const nodeTimingEntries = [];
-    for (const node of this._nodeTimings.getNodes()) {
-      const timing = this._nodeTimings.getCompleted(node);
-      nodeTimingEntries.push([node, {
+    const nodeTimingEntries = completeNodeTimingEntries.map(([node, timing]) => {
+      return [node, {
         startTime: timing.startTime,
         endTime: timing.endTime,
         duration: timing.endTime - timing.startTime,
-      }]);
-    }
+      }];
+    });
 
-    // Most consumers will want the entries sorted by startTime, so insert them in that order
-    nodeTimingEntries.sort((a, b) => a[1].startTime - b[1].startTime);
-    return new Map(nodeTimingEntries);
+    return {
+      nodeTimings: new Map(nodeTimingEntries),
+      completeNodeTimings: new Map(completeNodeTimingEntries),
+    };
   }
 
   /**
@@ -481,8 +492,9 @@ class Simulator {
       }
     }
 
-    const nodeTimings = this._computeFinalNodeTimings();
-    ALL_SIMULATION_NODE_TIMINGS.set(options.label || 'unlabeled', nodeTimings);
+    // `nodeTimings` are used for simulator consumers, `completeNodeTimings` kept for debugging.
+    const {nodeTimings, completeNodeTimings} = this._computeFinalNodeTimings();
+    ALL_SIMULATION_NODE_TIMINGS.set(options.label || 'unlabeled', completeNodeTimings);
 
     return {
       timeInMs: totalElapsedTime,
@@ -507,7 +519,7 @@ class Simulator {
     return wastedMs;
   }
 
-  /** @return {Map<string, LH.Gatherer.Simulation.Result['nodeTimings']>} */
+  /** @return {Map<string, Map<Node, CompleteNodeTiming>>} */
   static get ALL_NODE_TIMINGS() {
     return ALL_SIMULATION_NODE_TIMINGS;
   }

package/core/lib/dependency-graph/simulator/tcp-connection.js

@@ -4,6 +4,8 @@
  * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
  */
 
+/** @typedef {import('./simulator-timing-map.js').ConnectionTiming} ConnectionTiming */
+
 const INITIAL_CONGESTION_WINDOW = 10;
 const TCP_SEGMENT_SIZE = 1460;
 
@@ -176,12 +178,34 @@ class TcpConnection {
     const extraBytesDownloaded = this._h2 ? Math.max(totalBytesDownloaded - bytesToDownload, 0) : 0;
     const bytesDownloaded = Math.max(Math.min(totalBytesDownloaded, bytesToDownload), 0);
 
+    /** @type {ConnectionTiming} */
+    let connectionTiming;
+    if (!this._warmed) {
+      connectionTiming = {
+        dnsResolutionTime,
+        connectionTime: handshakeAndRequest - dnsResolutionTime,
+        sslTime: this._ssl ? twoWayLatency : undefined,
+        timeToFirstByte,
+      };
+    } else if (this._h2) {
+      // TODO: timing information currently difficult to model for warm h2 connections.
+      connectionTiming = {
+        timeToFirstByte,
+      };
+    } else {
+      connectionTiming = {
+        connectionTime: handshakeAndRequest,
+        timeToFirstByte,
+      };
+    }
+
     return {
       roundTrips,
       timeElapsed,
       bytesDownloaded,
       extraBytesDownloaded,
       congestionWindow,
+      connectionTiming,
     };
   }
 }
@@ -202,4 +226,5 @@ export {TcpConnection};
  * @property {number} bytesDownloaded
  * @property {number} extraBytesDownloaded
  * @property {number} congestionWindow
+ * @property {ConnectionTiming} connectionTiming
  */

package/core/lib/lantern-trace-saver.js

@@ -4,8 +4,11 @@
  * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
  */
 
+/** @typedef {import('./dependency-graph/base-node.js').Node} Node */
+/** @typedef {import('./dependency-graph/simulator/simulator.js').CompleteNodeTiming} CompleteNodeTiming */
+
 /**
- * @param {LH.Gatherer.Simulation.Result['nodeTimings']} nodeTimings
+ * @param {Map<Node, CompleteNodeTiming>} nodeTimings
  * @return {LH.Trace}
  */
 function convertNodeTimingsToTrace(nodeTimings) {
@@ -31,7 +34,8 @@ function convertNodeTimingsToTrace(nodeTimings) {
     } else {
       // Ignore data URIs as they don't really add much value
       if (/^data/.test(node.record.url)) continue;
-      traceEvents.push(...createFakeNetworkEvents(node.record, timing));
+      traceEvents.push(...createFakeNetworkEvents(requestId, node.record, timing));
+      requestId++;
     }
   }
 
@@ -117,12 +121,34 @@ function convertNodeTimingsToTrace(nodeTimings) {
   }
 
   /**
+   * @param {number} requestId
+   * @param {LH.Artifacts.NetworkRequest} record
+   * @param {CompleteNodeTiming} timing
+   * @return {LH.TraceEvent}
+   */
+  function createWillSendRequestEvent(requestId, record, timing) {
+    return {
+      ...baseEvent,
+      ph: 'I',
+      s: 't',
+      // No `dur` on network instant events but add to keep types happy.
+      dur: 0,
+      name: 'ResourceWillSendRequest',
+      ts: toMicroseconds(timing.startTime),
+      args: {data: {requestId: String(requestId)}},
+    };
+  }
+
+  /**
+   * @param {number} requestId
    * @param {LH.Artifacts.NetworkRequest} record
-   * @param {LH.Gatherer.Simulation.NodeTiming} timing
+   * @param {CompleteNodeTiming} timing
    * @return {LH.TraceEvent[]}
    */
-  function createFakeNetworkEvents(record, timing) {
-    requestId++;
+  function createFakeNetworkEvents(requestId, record, timing) {
+    if (!('connectionTiming' in timing)) {
+      throw new Error('Network node timing incomplete');
+    }
 
     // 0ms requests get super-messed up rendering
     // Use 0.3ms instead so they're still hoverable, https://github.com/GoogleChrome/lighthouse/pull/5350#discussion_r194563201
@@ -130,6 +156,7 @@ function convertNodeTimingsToTrace(nodeTimings) {
     if (startTime === endTime) endTime += 0.3;
 
     const requestData = {requestId: requestId.toString(), frame};
+    // No `dur` on network instant events but add to keep types happy.
     /** @type {LH.Util.StrictOmit<LH.TraceEvent, 'name'|'ts'|'args'>} */
     const baseRequestEvent = {...baseEvent, ph: 'I', s: 't', dur: 0};
 
@@ -140,6 +167,13 @@ function convertNodeTimingsToTrace(nodeTimings) {
       priority: record.priority,
     };
 
+    const {dnsResolutionTime, connectionTime, sslTime, timeToFirstByte} = timing.connectionTiming;
+    let sslStart = -1;
+    let sslEnd = -1;
+    if (connectionTime !== undefined && sslTime !== undefined) {
+      sslStart = connectionTime - sslTime;
+      sslEnd = connectionTime;
+    }
     const receiveResponseData = {
       ...requestData,
       statusCode: record.statusCode,
@@ -147,17 +181,45 @@ function convertNodeTimingsToTrace(nodeTimings) {
       encodedDataLength: record.transferSize,
       fromCache: record.fromDiskCache,
       fromServiceWorker: record.fetchedViaServiceWorker,
+      timing: {
+        // `requestTime` is in seconds.
+        requestTime: toMicroseconds(startTime) / (1000 * 1000),
+        // Remaining values are milliseconds after `requestTime`.
+        dnsStart: dnsResolutionTime === undefined ? -1 : 0,
+        dnsEnd: dnsResolutionTime ?? -1,
+        connectStart: dnsResolutionTime ?? -1,
+        connectEnd: connectionTime ?? -1,
+        sslStart,
+        sslEnd,
+        sendStart: connectionTime ?? 0,
+        sendEnd: connectionTime ?? 0,
+        receiveHeadersEnd: timeToFirstByte,
+        workerStart: -1,
+        workerReady: -1,
+        proxyStart: -1,
+        proxyEnd: -1,
+        pushStart: 0,
+        pushEnd: 0,
+      },
     };
 
     const resourceFinishData = {
-      ...requestData,
+      requestId: requestId.toString(),
+      encodedDataLength: record.transferSize,
       decodedBodyLength: record.resourceSize,
       didFail: !!record.failed,
       finishTime: toMicroseconds(endTime) / (1000 * 1000),
     };
 
     /** @type {LH.TraceEvent[]} */
-    const events = [
+    const events = [];
+
+    // Navigation request needs an additional ResourceWillSendRequest event.
+    if (requestId === 1) {
+      events.push(createWillSendRequestEvent(requestId, record, timing));
+    }
+
+    events.push(
       {
         ...baseRequestEvent,
         name: 'ResourceSendRequest',
@@ -169,13 +231,14 @@ function convertNodeTimingsToTrace(nodeTimings) {
         name: 'ResourceFinish',
         ts: toMicroseconds(endTime),
         args: {data: resourceFinishData},
-      },
-    ];
+      }
+    );
 
     if (!record.failed) {
       events.push({
         ...baseRequestEvent,
         name: 'ResourceReceiveResponse',
+        // Event `ts` isn't meaningful, so just pick a time.
         ts: toMicroseconds((startTime + endTime) / 2),
         args: {data: receiveResponseData},
       });

package/core/lib/third-party-web.js

@@ -4,7 +4,7 @@
  * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
  */
 
-import thirdPartyWeb from 'third-party-web/httparchive-nostats-subset.js';
+import thirdPartyWeb from 'third-party-web/nostats-subset.js';
 
 /** @typedef {import("third-party-web").IEntity} ThirdPartyEntity */
 /** @typedef {import("third-party-web").IProduct} ThirdPartyProduct */

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "lighthouse",
   "type": "module",
-  "version": "9.5.0-dev.20221129",
+  "version": "9.5.0-dev.20221201",
   "description": "Automated auditing, performance metrics, and best practices for the web.",
   "main": "./core/index.js",
   "bin": {
@@ -206,7 +206,7 @@
     "robots-parser": "^3.0.0",
     "semver": "^5.3.0",
     "speedline-core": "^1.4.3",
-    "third-party-web": "^0.17.1",
+    "third-party-web": "^0.20.2",
     "ws": "^7.0.0",
     "yargs": "^17.3.1",
     "yargs-parser": "^21.0.0"