lighthouse 8.0.0-dev.20210709 → 8.0.0

package/{report/report-assets.js → lighthouse-core/lib/sd-validation/helpers/walk-object.js}

@@ -5,18 +5,26 @@
  */
 'use strict';
 
-const fs = require('fs');
+/**
+ * Recursively (DFS) traverses an object and calls provided function for each field.
+ *
+ * @param {*} obj
+ * @param {function(string, any, Array<string>, any): void} callback
+ * @param {Array<string>} path
+ */
+module.exports = function walkObject(obj, callback, path = []) {
+  if (obj === null) {
+    return;
+  }
+
+  Object.entries(obj).forEach(([fieldName, fieldValue]) => {
+    const newPath = Array.from(path);
+    newPath.push(fieldName);
 
-const REPORT_TEMPLATE = fs.readFileSync(__dirname + '/assets/standalone-template.html', 'utf8');
-const REPORT_JAVASCRIPT = fs.readFileSync(__dirname + '/../dist/report/standalone.js', 'utf8');
-const REPORT_CSS = fs.readFileSync(__dirname + '/assets/styles.css', 'utf8');
-const REPORT_TEMPLATES = fs.readFileSync(__dirname + '/assets/templates.html', 'utf8');
+    callback(fieldName, fieldValue, newPath, obj);
 
-// Changes to this export interface should be reflected in build/build-dt-report-resources.js
-// and clients/devtools-report-assets.js
-module.exports = {
-  REPORT_TEMPLATE,
-  REPORT_TEMPLATES,
-  REPORT_JAVASCRIPT,
-  REPORT_CSS,
+    if (typeof fieldValue === 'object') {
+      walkObject(fieldValue, callback, newPath);
+    }
+  });
 };

package/lighthouse-core/lib/sd-validation/json-expander.js

@@ -0,0 +1,56 @@
+/**
+ * @license Copyright 2018 The Lighthouse Authors. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const {URL} = require('../url-shim.js');
+const jsonld = require('jsonld');
+const schemaOrgContext = require('./assets/jsonldcontext.json');
+const SCHEMA_ORG_HOST = 'schema.org';
+
+/**
+ * Custom loader that prevents network calls and allows us to return local version of the
+ * schema.org document
+ * @param {string} schemaUrl
+ * @return {Promise<import('jsonld').RemoteDocument>}
+ */
+async function documentLoader(schemaUrl) {
+  let urlObj = null;
+
+  try {
+    // Give a dummy base URL so relative URLs will be considered valid.
+    urlObj = new URL(schemaUrl, 'http://example.com');
+  } catch (e) {
+    throw new Error('Error parsing URL: ' + schemaUrl);
+  }
+
+  if (urlObj.host === SCHEMA_ORG_HOST && urlObj.pathname === '/') {
+    return {
+      document: schemaOrgContext,
+    };
+  } else {
+    // We only process schema.org, for other schemas we return an empty object
+    return {
+      document: {},
+    };
+  }
+}
+
+/**
+ * Takes JSON-LD object and normalizes it by following the expansion algorithm
+ * (https://json-ld.org/spec/latest/json-ld-api/#expansion).
+ *
+ * @param {any} inputObject
+ * @returns {Promise<LH.StructuredData.ExpandedSchemaRepresentation|null>}
+ */
+module.exports = async function expand(inputObject) {
+  try {
+    return await jsonld.expand(inputObject, {documentLoader});
+  } catch (err) {
+    // jsonld wraps real errors in a bunch of junk, so see we have an underlying error first
+    if (err.details && err.details.cause) throw err.details.cause;
+    throw err;
+  }
+};

package/lighthouse-core/lib/sd-validation/json-linter.js

@@ -0,0 +1,49 @@
+/**
+ * @license Copyright 2018 The Lighthouse Authors. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const jsonlint = require('jsonlint-mod');
+
+/**
+ * @param {string} input
+ * @returns {{message: string, lineNumber: number|null}|null}
+ */
+module.exports = function parseJSON(input) {
+  try {
+    jsonlint.parse(input);
+  } catch (error) {
+    /** @type {number|null} */
+    let line = error.at;
+
+    // extract line number from message
+    if (!line) {
+      const regexLineResult = error.message.match(/Parse error on line (\d+)/);
+
+      if (regexLineResult) {
+        line = parseFloat(regexLineResult[1]);
+      }
+    }
+
+
+    // jsonlint error message points to a specific character, but we just want the message.
+    // Example:
+    //    ---------^
+    //    Unexpected character {
+    let message = /** @type {string} */ (error.message);
+    const regexMessageResult = error.message.match(/-+\^\n(.+)$/);
+
+    if (regexMessageResult) {
+      message = regexMessageResult[1];
+    }
+
+    return {
+      message,
+      lineNumber: line,
+    };
+  }
+
+  return null;
+};

package/lighthouse-core/lib/sd-validation/jsonld-keyword-validator.js

@@ -0,0 +1,50 @@
+/**
+ * @license Copyright 2018 The Lighthouse Authors. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const walkObject = require('./helpers/walk-object.js');
+
+// This list comes from the JSON-LD 1.1 editors draft:
+// https://w3c.github.io/json-ld-syntax/#syntax-tokens-and-keywords
+const VALID_KEYWORDS = new Set([
+  '@base',
+  '@container',
+  '@context',
+  '@graph',
+  '@id',
+  '@index',
+  '@language',
+  '@list',
+  '@nest',
+  '@none',
+  '@prefix',
+  '@reverse',
+  '@set',
+  '@type',
+  '@value',
+  '@version',
+  '@vocab',
+]);
+
+/**
+ * @param {*} json
+ * @return {Array<{path: string, message: string}>}
+ */
+module.exports = function validateJsonLD(json) {
+  /** @type {Array<{path: string, message: string}>} */
+  const errors = [];
+
+  walkObject(json, (name, value, path) => {
+    if (name.startsWith('@') && !VALID_KEYWORDS.has(name)) {
+      errors.push({
+        path: path.join('/'),
+        message: `Unknown keyword "${name}"`,
+      });
+    }
+  });
+
+  return errors;
+};

package/lighthouse-core/lib/sd-validation/line-number-from-jsonld-path.js

@@ -0,0 +1,60 @@
+/**
+ * @license Copyright 2019 The Lighthouse Authors. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+/**
+ * @param {*} obj - JSON-LD object
+ * @param {string} path - slash-separated path in the JSON-LD object (uses relative URIs)
+ * @returns {null | number} - line number of the path value in the prettified JSON
+ */
+function getLineNumberFromJsonLDPath(obj, path) {
+  // To avoid having an extra dependency on a JSON parser we set a unique key in the
+  // object and then use that to identify the correct line
+  const searchKey = Math.random().toString();
+  obj = JSON.parse(JSON.stringify(obj));
+
+  setValueAtJsonLDPath(obj, path, searchKey);
+  const jsonLines = JSON.stringify(obj, null, 2).split('\n');
+  const lineIndex = jsonLines.findIndex(line => line.includes(searchKey));
+
+  return lineIndex === -1 ? null : lineIndex + 1;
+}
+
+/**
+ * @param {*} obj
+ * @param {string} path
+ * @param {*} value
+ */
+function setValueAtJsonLDPath(obj, path, value) {
+  const pathParts = path.split('/').filter(p => !!p);
+  let currentObj = obj;
+  pathParts.forEach((pathPart, i) => {
+    if (pathPart === '0' && !Array.isArray(currentObj)) {
+      // jsonld expansion turns single values into arrays
+      return;
+    }
+
+    const isLastPart = pathParts.length - 1 === i;
+    for (const key of Object.keys(currentObj)) {
+      // The actual key in JSON might be an absolute IRI like "http://schema.org/author"
+      // but key provided by validator is "author"
+      const keyParts = key.split('/');
+      const relativeKey = keyParts[keyParts.length - 1];
+      if (relativeKey === pathPart) {
+        // If we've arrived at the end of the provided path set the value, otherwise
+        // continue iterating with the object at the key location
+        if (isLastPart) {
+          currentObj[key] = value;
+        } else {
+          currentObj = currentObj[key];
+        }
+        return;
+      }
+    }
+  });
+}
+
+module.exports = getLineNumberFromJsonLDPath;

package/lighthouse-core/lib/sd-validation/schema-validator.js

@@ -0,0 +1,146 @@
+/**
+ * @license Copyright 2018 The Lighthouse Authors. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const walkObject = require('./helpers/walk-object.js');
+const schemaStructure = require('./assets/schema-tree.json');
+const TYPE_KEYWORD = '@type';
+const SCHEMA_ORG_URL_REGEX = /https?:\/\/schema\.org\//;
+
+/**
+ * @param {string} uri
+ * @returns {string}
+ */
+function cleanName(uri) {
+  return uri.replace(SCHEMA_ORG_URL_REGEX, '');
+}
+
+/**
+ * @param {string} type
+ * @returns {Array<string>}
+ */
+function getPropsForType(type) {
+  const cleanType = cleanName(type);
+  const props = schemaStructure.properties
+    .filter(prop => prop.parent.includes(cleanType))
+    .map(prop => prop.name);
+  const foundType = findType(type);
+  if (!foundType) throw new Error(`Unable to get props for missing type "${type}"`);
+  const parentTypes = foundType.parent;
+
+  return parentTypes.reduce((allProps, type) => allProps.concat(getPropsForType(type)), props);
+}
+
+/**
+ * @param {string} type
+ * @returns {{name: string, parent: Array<string>}|undefined}
+ */
+function findType(type) {
+  const cleanType = cleanName(type);
+
+  return schemaStructure.types.find(typeObj => typeObj.name === cleanType);
+}
+
+/**
+ * Validates keys of given object based on its type(s). Returns an array of error messages.
+ *
+ * @param {string|Array<string>} typeOrTypes
+ * @param {Array<string>} keys
+ * @returns {Array<Pick<LH.StructuredData.ValidationError, "message" | "validTypes"> & {key?: string}>}
+ */
+function validateObjectKeys(typeOrTypes, keys) {
+  /** @type {Array<string>} */
+  let types = [];
+
+  if (typeof typeOrTypes === 'string') {
+    types = [typeOrTypes];
+  } else if (Array.isArray(typeOrTypes)) {
+    types = typeOrTypes;
+    const invalidIndex = typeOrTypes.findIndex(s => typeof s !== 'string');
+    if (invalidIndex >= 0) {
+      return [{message: `Unknown value type at index ${invalidIndex}`}];
+    }
+  } else {
+    return [{message: 'Unknown value type'}];
+  }
+
+  const unknownTypes = types.filter(t => !findType(t));
+
+  if (unknownTypes.length) {
+    return unknownTypes
+      .filter(type => SCHEMA_ORG_URL_REGEX.test(type))
+      .map(type => ({
+        message: `Unrecognized schema.org type: ${type}`,
+        key: '@type',
+      }));
+  }
+
+  /** @type {Set<string>} */
+  const allKnownProps = new Set();
+
+  types.forEach(type => {
+    const knownProps = getPropsForType(type);
+
+    knownProps.forEach(key => allKnownProps.add(key));
+  });
+
+  const cleanKeys = keys
+    // Skip JSON-LD keywords (including invalid ones as they were already flagged in the json-ld validator)
+    .filter(key => key.indexOf('@') !== 0)
+    .map(key => cleanName(key));
+
+  return cleanKeys
+    // remove Schema.org input/output constraints http://schema.org/docs/actions.html#part-4
+    .map(key => key.replace(/-(input|output)$/, ''))
+    .filter(key => !allKnownProps.has(key))
+    .map(key => ({
+      message: `Unexpected property "${key}"`,
+      key,
+      validTypes: types,
+    }));
+}
+
+/**
+ * @param {LH.StructuredData.ExpandedSchemaRepresentation|null} expandedObj Valid JSON-LD object in expanded form
+ * @return {Array<Pick<LH.StructuredData.ValidationError, "message" | "validTypes" | "path">>}
+ */
+module.exports = function validateSchemaOrg(expandedObj) {
+  /** @type {Array<Pick<LH.StructuredData.ValidationError, "message" | "validTypes" | "path">>} */
+  const errors = [];
+
+  if (expandedObj === null) {
+    return errors;
+  }
+
+  // If the array only has a single item, treat it as if it was at the root to simplify the error path.
+  // Arrays longer than a single item are handled in `walkObject` below.
+  if (Array.isArray(expandedObj) && expandedObj.length === 1) {
+    expandedObj = expandedObj[0];
+  }
+
+  walkObject(expandedObj, (name, value, path, obj) => {
+    if (name === TYPE_KEYWORD) {
+      const keyErrors = validateObjectKeys(value, Object.keys(obj));
+
+      keyErrors.forEach(error => {
+        errors.push({
+          validTypes: error.validTypes,
+          message: error.message,
+          // get rid of the last chunk (/@type) as it's the same for all errors
+          path:
+            '/' +
+            path
+              .slice(0, -1)
+              .concat(error.key || [])
+              .map(cleanName)
+              .join('/'),
+        });
+      });
+    }
+  });
+
+  return errors;
+};

package/lighthouse-core/lib/sd-validation/scripts/download-jsonldcontext.js

@@ -0,0 +1,30 @@
+/**
+ * @license Copyright 2018 The Lighthouse Authors. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+/**
+ * Call this script to update assets/jsonldcontext.json with the latest schema.org spec
+ */
+
+const fetch = require('node-fetch');
+const path = require('path');
+const fs = require('fs');
+
+const SCHEMA_ORG_URL = 'https://schema.org';
+const CONTEXT_FILE = path.join(__dirname, '../assets/jsonldcontext.json');
+
+async function run() {
+  try {
+    const response = await fetch(SCHEMA_ORG_URL, {headers: {Accept: 'application/ld+json'}});
+    const data = await response.json();
+    fs.writeFileSync(CONTEXT_FILE, JSON.stringify(data, null, 2));
+    console.log('Success.'); // eslint-disable-line no-console
+  } catch (e) {
+    console.error(e); // eslint-disable-line no-console
+  }
+}
+
+run();

package/lighthouse-core/lib/sd-validation/scripts/generate-schema-tree.js

@@ -0,0 +1,97 @@
+/**
+ * @license Copyright 2018 The Lighthouse Authors. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+/**
+ * Call this script to update assets/schema-tree.json with the latest schema.org spec
+ */
+
+const fetch = require('node-fetch');
+const path = require('path');
+const fs = require('fs');
+
+const SCHEMA_ORG_URL = 'https://schema.org/version/latest/schema.jsonld';
+const SCHEMA_TREE_FILE = path.join(__dirname, '../assets/schema-tree.json');
+
+/** @typedef {import('jsonlint-mod').SchemaTreeItem} SchemaDefinition */
+
+/** @typedef {import('jsonlint-mod').JSONSchemaSource} SchemaSource */
+
+/** @typedef {{'@id': string}} IDRef */
+
+/**
+ * @param {SchemaSource} data
+ */
+function processData(data) {
+  /** @type {SchemaDefinition[]} */
+  const types = [];
+  /** @type {SchemaDefinition[]} */
+  const properties = [];
+
+  /** @param {string} str */
+  function removePrefix(str) {
+    return str.replace('http://schema.org/', '');
+  }
+
+  /**
+   * Accepts some set of id references and returns the array of cleaned references without the `http://schema.org` prefix.
+   * @param {Array<IDRef>|IDRef|undefined} parents
+   */
+  function cleanIdPrefixes(parents) {
+    if (Array.isArray(parents)) {
+      return parents.map(item => removePrefix(item['@id']));
+    } else if (parents && parents['@id']) {
+      return [removePrefix(parents['@id'])];
+    }
+
+    return [];
+  }
+
+  // Go through all the graph entries to find the valid types and properties.
+  // i.e. this converts
+  // [
+  //   {@id: 'http://schema.org/CafeOrCoffeeShop', @type: 'rdfs:Class', rdfs:subClassOf: {@id: 'http://schema.org/FoodEstablishment'}},
+  //   {@id: 'http://schema.org/bestRating', @type: 'rdfs:Property', http://schema.org/domainIncludes: {@id: 'http://schema.org/Rating'}},
+  // ]
+  // into
+  // {
+  //   types: [{name: 'CafeOrCoffeeShop', parent: ['FoodEstablishment']}],
+  //   properties: [{name: 'bestRating', parent: ['Rating']}],
+  // }
+  data['@graph'].forEach(item => {
+    if (item['rdfs:label'] === undefined) {
+      return;
+    }
+
+    if (item['@type'] === 'rdf:Property') {
+      properties.push({
+        name: item['rdfs:label'],
+        parent: cleanIdPrefixes(item['http://schema.org/domainIncludes']),
+      });
+    } else {
+      types.push({
+        name: item['rdfs:label'],
+        parent: cleanIdPrefixes(item['rdfs:subClassOf']),
+      });
+    }
+  });
+
+  return {types, properties};
+}
+
+async function run() {
+  try {
+    const response = await fetch(SCHEMA_ORG_URL);
+    const data = await response.json();
+    const processed = processData(data);
+    fs.writeFileSync(SCHEMA_TREE_FILE, JSON.stringify(processed, null, 2));
+    console.log('Success.'); // eslint-disable-line no-console
+  } catch (e) {
+    console.error(e); // eslint-disable-line no-console
+  }
+}
+
+run();

package/lighthouse-core/lib/sd-validation/sd-validation.js

@@ -0,0 +1,77 @@
+/**
+ * @license Copyright 2018 The Lighthouse Authors. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const parseJSON = require('./json-linter.js');
+const validateJsonLD = require('./jsonld-keyword-validator.js');
+const expandAsync = require('./json-expander.js');
+const validateSchemaOrg = require('./schema-validator.js');
+const getLineNumberFromJsonLDPath = require('./line-number-from-jsonld-path.js');
+
+/**
+ * Validates JSON-LD input. Returns array of error objects.
+ *
+ * @param {string} textInput
+ * @returns {Promise<Array<LH.StructuredData.ValidationError>>}
+ */
+module.exports = async function validate(textInput) {
+  // STEP 1: VALIDATE JSON
+  const parseError = parseJSON(textInput);
+
+  if (parseError) {
+    return [{
+      validator: 'json',
+      lineNumber: parseError.lineNumber,
+      message: parseError.message,
+    }];
+  }
+
+  const inputObject = JSON.parse(textInput);
+
+  // STEP 2: VALIDATE JSONLD
+  const jsonLdErrors = validateJsonLD(inputObject);
+
+  if (jsonLdErrors.length) {
+    return jsonLdErrors.map(error => {
+      return {
+        validator: 'json-ld',
+        path: error.path,
+        message: error.message,
+        lineNumber: getLineNumberFromJsonLDPath(inputObject, error.path),
+      };
+    });
+  }
+
+  // STEP 3: EXPAND
+  /** @type {LH.StructuredData.ExpandedSchemaRepresentation|null} */
+  let expandedObj = null;
+  try {
+    expandedObj = await expandAsync(inputObject);
+  } catch (error) {
+    return [{
+      validator: 'json-ld-expand',
+      message: error.message,
+    }];
+  }
+
+  // STEP 4: VALIDATE SCHEMA
+  const schemaOrgErrors = validateSchemaOrg(expandedObj);
+
+  if (schemaOrgErrors.length) {
+    return schemaOrgErrors.map(error => {
+      return {
+        validator: 'schema-org',
+        path: error.path,
+        message: error.message,
+        lineNumber: error.path ? getLineNumberFromJsonLDPath(inputObject, error.path) : null,
+        validTypes: error.validTypes,
+      };
+    });
+  }
+
+  return [];
+};
+

package/lighthouse-core/lib/tappable-rects.js

@@ -44,7 +44,7 @@ function almostEqual(a, b) {
 /**
  * Merge touching rects based on what appears as one tappable area to the user.
  * @param {LH.Artifacts.Rect[]} clientRects
- * @return {LH.Artifacts.Rect[]}
+ * @returns {LH.Artifacts.Rect[]}
  */
 function mergeTouchingClientRects(clientRects) {
   for (let i = 0; i < clientRects.length; i++) {