lighthouse 9.5.0-dev.20230111 → 9.5.0-dev.20230112
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/bin.js +6 -6
- package/cli/run.js +1 -1
- package/cli/test/smokehouse/lighthouse-runners/bundle.js +9 -9
- package/cli/test/smokehouse/lighthouse-runners/cli.js +7 -7
- package/cli/test/smokehouse/lighthouse-runners/devtools.js +3 -3
- package/cli/test/smokehouse/readme.md +1 -1
- package/cli/test/smokehouse/smokehouse.js +13 -13
- package/core/audits/seo/is-crawlable.js +114 -42
- package/core/config/config-helpers.js +11 -11
- package/core/config/config-plugin.js +2 -2
- package/core/config/config.js +15 -15
- package/core/config/default-config.js +2 -2
- package/core/config/desktop-config.js +1 -1
- package/core/config/experimental-config.js +1 -1
- package/core/config/full-config.js +1 -1
- package/core/config/lr-desktop-config.js +1 -1
- package/core/config/lr-mobile-config.js +1 -1
- package/core/config/perf-config.js +1 -1
- package/core/config/validation.js +4 -4
- package/core/gather/navigation-runner.js +1 -1
- package/core/gather/snapshot-runner.js +1 -1
- package/core/gather/timespan-runner.js +1 -1
- package/core/index.js +10 -10
- package/core/legacy/config/config.js +23 -23
- package/core/legacy/config/legacy-default-config.js +1 -1
- package/core/user-flow.js +3 -3
- package/core/util.cjs +13 -3
- package/dist/report/bundle.esm.js +13 -3
- package/dist/report/flow.js +1 -1
- package/dist/report/standalone.js +1 -1
- package/package.json +1 -1
- package/report/renderer/util.js +13 -3
- package/report/test/generator/report-generator-test.js +1 -1
- package/report/test/renderer/util-test.js +2 -0
- package/types/config.d.ts +18 -18
- package/types/smokehouse.d.ts +2 -2
- package/types/user-flow.d.ts +1 -1
package/cli/bin.js
CHANGED
|
@@ -63,20 +63,20 @@ async function begin() {
|
|
|
63
63
|
|
|
64
64
|
const urlUnderTest = cliFlags._[0];
|
|
65
65
|
|
|
66
|
-
/** @type {LH.Config
|
|
67
|
-
let
|
|
66
|
+
/** @type {LH.Config|undefined} */
|
|
67
|
+
let config;
|
|
68
68
|
if (cliFlags.configPath) {
|
|
69
69
|
// Resolve the config file path relative to where cli was called.
|
|
70
70
|
cliFlags.configPath = path.resolve(process.cwd(), cliFlags.configPath);
|
|
71
71
|
|
|
72
72
|
if (cliFlags.configPath.endsWith('.json')) {
|
|
73
|
-
|
|
73
|
+
config = JSON.parse(fs.readFileSync(cliFlags.configPath, 'utf-8'));
|
|
74
74
|
} else {
|
|
75
75
|
const configModuleUrl = url.pathToFileURL(cliFlags.configPath).href;
|
|
76
|
-
|
|
76
|
+
config = (await import(configModuleUrl)).default;
|
|
77
77
|
}
|
|
78
78
|
} else if (cliFlags.preset) {
|
|
79
|
-
|
|
79
|
+
config = (await import(`../core/config/${cliFlags.preset}-config.js`)).default;
|
|
80
80
|
}
|
|
81
81
|
|
|
82
82
|
if (cliFlags.budgetPath) {
|
|
@@ -132,7 +132,7 @@ async function begin() {
|
|
|
132
132
|
});
|
|
133
133
|
}
|
|
134
134
|
|
|
135
|
-
return runLighthouse(urlUnderTest, cliFlags,
|
|
135
|
+
return runLighthouse(urlUnderTest, cliFlags, config);
|
|
136
136
|
}
|
|
137
137
|
|
|
138
138
|
export {
|
package/cli/run.js
CHANGED
|
@@ -209,7 +209,7 @@ async function potentiallyKillChrome(launchedChrome) {
|
|
|
209
209
|
/**
|
|
210
210
|
* @param {string} url
|
|
211
211
|
* @param {LH.CliFlags} flags
|
|
212
|
-
* @param {LH.Config
|
|
212
|
+
* @param {LH.Config|undefined} config
|
|
213
213
|
* @return {Promise<LH.RunnerResult|undefined>}
|
|
214
214
|
*/
|
|
215
215
|
async function runLighthouse(url, flags, config) {
|
|
@@ -26,9 +26,9 @@ import {loadArtifacts, saveArtifacts} from '../../../../core/lib/asset-saver.js'
|
|
|
26
26
|
// This runs only in the worker. The rest runs on the main thread.
|
|
27
27
|
if (!isMainThread && parentPort) {
|
|
28
28
|
(async () => {
|
|
29
|
-
const {url,
|
|
29
|
+
const {url, config, testRunnerOptions} = workerData;
|
|
30
30
|
try {
|
|
31
|
-
const result = await runBundledLighthouse(url,
|
|
31
|
+
const result = await runBundledLighthouse(url, config, testRunnerOptions);
|
|
32
32
|
// Save to assets directory because LighthouseError won't survive postMessage.
|
|
33
33
|
const assetsDir = fs.mkdtempSync(os.tmpdir() + '/smoke-bundle-assets-');
|
|
34
34
|
await saveArtifacts(result.artifacts, assetsDir);
|
|
@@ -46,11 +46,11 @@ if (!isMainThread && parentPort) {
|
|
|
46
46
|
|
|
47
47
|
/**
|
|
48
48
|
* @param {string} url
|
|
49
|
-
* @param {LH.Config
|
|
49
|
+
* @param {LH.Config|undefined} config
|
|
50
50
|
* @param {{isDebug?: boolean, useLegacyNavigation?: boolean}} testRunnerOptions
|
|
51
51
|
* @return {Promise<{lhr: LH.Result, artifacts: LH.Artifacts}>}
|
|
52
52
|
*/
|
|
53
|
-
async function runBundledLighthouse(url,
|
|
53
|
+
async function runBundledLighthouse(url, config, testRunnerOptions) {
|
|
54
54
|
if (isMainThread || !parentPort) {
|
|
55
55
|
throw new Error('must be called in worker');
|
|
56
56
|
}
|
|
@@ -87,12 +87,12 @@ async function runBundledLighthouse(url, configJson, testRunnerOptions) {
|
|
|
87
87
|
if (testRunnerOptions.useLegacyNavigation) {
|
|
88
88
|
const connection = new CriConnection(port);
|
|
89
89
|
runnerResult =
|
|
90
|
-
await legacyNavigation(url, {port, logLevel},
|
|
90
|
+
await legacyNavigation(url, {port, logLevel}, config, connection);
|
|
91
91
|
} else {
|
|
92
92
|
// Puppeteer is not included in the bundle, we must create the page here.
|
|
93
93
|
const browser = await puppeteer.connect({browserURL: `http://localhost:${port}`});
|
|
94
94
|
const page = await browser.newPage();
|
|
95
|
-
runnerResult = await lighthouse(url, {port, logLevel},
|
|
95
|
+
runnerResult = await lighthouse(url, {port, logLevel}, config, page);
|
|
96
96
|
}
|
|
97
97
|
if (!runnerResult) throw new Error('No runnerResult');
|
|
98
98
|
|
|
@@ -109,17 +109,17 @@ async function runBundledLighthouse(url, configJson, testRunnerOptions) {
|
|
|
109
109
|
/**
|
|
110
110
|
* Launch Chrome and do a full Lighthouse run via the Lighthouse DevTools bundle.
|
|
111
111
|
* @param {string} url
|
|
112
|
-
* @param {LH.Config
|
|
112
|
+
* @param {LH.Config=} config
|
|
113
113
|
* @param {{isDebug?: boolean, useLegacyNavigation?: boolean}=} testRunnerOptions
|
|
114
114
|
* @return {Promise<{lhr: LH.Result, artifacts: LH.Artifacts, log: string}>}
|
|
115
115
|
*/
|
|
116
|
-
async function runLighthouse(url,
|
|
116
|
+
async function runLighthouse(url, config, testRunnerOptions = {}) {
|
|
117
117
|
/** @type {string[]} */
|
|
118
118
|
const logs = [];
|
|
119
119
|
const worker = new Worker(new URL(import.meta.url), {
|
|
120
120
|
stdout: true,
|
|
121
121
|
stderr: true,
|
|
122
|
-
workerData: {url,
|
|
122
|
+
workerData: {url, config, testRunnerOptions},
|
|
123
123
|
});
|
|
124
124
|
worker.stdout.setEncoding('utf8');
|
|
125
125
|
worker.stderr.setEncoding('utf8');
|
|
@@ -27,16 +27,16 @@ const execFileAsync = promisify(execFile);
|
|
|
27
27
|
/**
|
|
28
28
|
* Launch Chrome and do a full Lighthouse run via the Lighthouse CLI.
|
|
29
29
|
* @param {string} url
|
|
30
|
-
* @param {LH.Config
|
|
30
|
+
* @param {LH.Config=} config
|
|
31
31
|
* @param {{isDebug?: boolean, useFraggleRock?: boolean}=} testRunnerOptions
|
|
32
32
|
* @return {Promise<{lhr: LH.Result, artifacts: LH.Artifacts, log: string}>}
|
|
33
33
|
*/
|
|
34
|
-
async function runLighthouse(url,
|
|
34
|
+
async function runLighthouse(url, config, testRunnerOptions = {}) {
|
|
35
35
|
const {isDebug} = testRunnerOptions;
|
|
36
36
|
const tmpDir = `${LH_ROOT}/.tmp/smokehouse`;
|
|
37
37
|
await fs.mkdir(tmpDir, {recursive: true});
|
|
38
38
|
const tmpPath = await fs.mkdtemp(`${tmpDir}/smokehouse-`);
|
|
39
|
-
return internalRun(url, tmpPath,
|
|
39
|
+
return internalRun(url, tmpPath, config, testRunnerOptions)
|
|
40
40
|
// Wait for internalRun() before removing scratch directory.
|
|
41
41
|
.finally(() => !isDebug && fs.rm(tmpPath, {recursive: true, force: true}));
|
|
42
42
|
}
|
|
@@ -45,11 +45,11 @@ async function runLighthouse(url, configJson, testRunnerOptions = {}) {
|
|
|
45
45
|
* Internal runner.
|
|
46
46
|
* @param {string} url
|
|
47
47
|
* @param {string} tmpPath
|
|
48
|
-
* @param {LH.Config
|
|
48
|
+
* @param {LH.Config=} config
|
|
49
49
|
* @param {{isDebug?: boolean, useLegacyNavigation?: boolean}=} options
|
|
50
50
|
* @return {Promise<{lhr: LH.Result, artifacts: LH.Artifacts, log: string}>}
|
|
51
51
|
*/
|
|
52
|
-
async function internalRun(url, tmpPath,
|
|
52
|
+
async function internalRun(url, tmpPath, config, options) {
|
|
53
53
|
const {isDebug = false, useLegacyNavigation = false} = options || {};
|
|
54
54
|
const localConsole = new LocalConsole();
|
|
55
55
|
|
|
@@ -72,9 +72,9 @@ async function internalRun(url, tmpPath, configJson, options) {
|
|
|
72
72
|
}
|
|
73
73
|
|
|
74
74
|
// Config can be optionally provided.
|
|
75
|
-
if (
|
|
75
|
+
if (config) {
|
|
76
76
|
const configPath = `${tmpPath}/config.json`;
|
|
77
|
-
await fs.writeFile(configPath, JSON.stringify(
|
|
77
|
+
await fs.writeFile(configPath, JSON.stringify(config));
|
|
78
78
|
args.push(`--config-path=${configPath}`);
|
|
79
79
|
}
|
|
80
80
|
|
|
@@ -41,16 +41,16 @@ async function setup() {
|
|
|
41
41
|
* unless DEVTOOLS_PATH is set.
|
|
42
42
|
* CHROME_PATH determines which Chrome is used–otherwise the default is puppeteer's chrome binary.
|
|
43
43
|
* @param {string} url
|
|
44
|
-
* @param {LH.Config
|
|
44
|
+
* @param {LH.Config=} config
|
|
45
45
|
* @param {{isDebug?: boolean, useLegacyNavigation?: boolean}=} testRunnerOptions
|
|
46
46
|
* @return {Promise<{lhr: LH.Result, artifacts: LH.Artifacts, log: string}>}
|
|
47
47
|
*/
|
|
48
|
-
async function runLighthouse(url,
|
|
48
|
+
async function runLighthouse(url, config, testRunnerOptions = {}) {
|
|
49
49
|
const chromeFlags = [
|
|
50
50
|
`--custom-devtools-frontend=file://${devtoolsDir}/out/LighthouseIntegration/gen/front_end`,
|
|
51
51
|
];
|
|
52
52
|
const {lhr, artifacts, logs} = await testUrlFromDevtools(url, {
|
|
53
|
-
config
|
|
53
|
+
config,
|
|
54
54
|
chromeFlags,
|
|
55
55
|
useLegacyNavigation: testRunnerOptions.useLegacyNavigation,
|
|
56
56
|
});
|
|
@@ -14,7 +14,7 @@ See [`SmokehouseOptions`](https://github.com/GoogleChrome/lighthouse/blob/main/c
|
|
|
14
14
|
| -------------- | ---------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- |
|
|
15
15
|
| `id` | `string` | The string identifier of the test. |
|
|
16
16
|
| `expectations` | `{lhr: Object, artifacts: Object}` | See below. |
|
|
17
|
-
| `config` | `LH.Config
|
|
17
|
+
| `config` | `LH.Config` (optional) | An optional Lighthouse config. If not specified, the default config is used. |
|
|
18
18
|
| `runSerially` | `boolean` (optional) | An optional flag. If set to true, the test won't be run in parallel to other tests. Useful if the test is performance sensitive. |
|
|
19
19
|
|
|
20
20
|
### Expectations
|
|
@@ -134,21 +134,21 @@ function purpleify(str) {
|
|
|
134
134
|
}
|
|
135
135
|
|
|
136
136
|
/**
|
|
137
|
-
* @param {LH.Config
|
|
138
|
-
* @return {LH.Config
|
|
137
|
+
* @param {LH.Config=} config
|
|
138
|
+
* @return {LH.Config|undefined}
|
|
139
139
|
*/
|
|
140
|
-
function convertToLegacyConfig(
|
|
141
|
-
if (!
|
|
140
|
+
function convertToLegacyConfig(config) {
|
|
141
|
+
if (!config) return config;
|
|
142
142
|
|
|
143
143
|
return {
|
|
144
|
-
...
|
|
144
|
+
...config,
|
|
145
145
|
passes: [{
|
|
146
146
|
passName: 'defaultPass',
|
|
147
|
-
pauseAfterFcpMs:
|
|
148
|
-
pauseAfterLoadMs:
|
|
149
|
-
networkQuietThresholdMs:
|
|
150
|
-
cpuQuietThresholdMs:
|
|
151
|
-
blankPage:
|
|
147
|
+
pauseAfterFcpMs: config.settings?.pauseAfterFcpMs,
|
|
148
|
+
pauseAfterLoadMs: config.settings?.pauseAfterLoadMs,
|
|
149
|
+
networkQuietThresholdMs: config.settings?.networkQuietThresholdMs,
|
|
150
|
+
cpuQuietThresholdMs: config.settings?.cpuQuietThresholdMs,
|
|
151
|
+
blankPage: config.settings?.blankPage,
|
|
152
152
|
}],
|
|
153
153
|
};
|
|
154
154
|
}
|
|
@@ -184,15 +184,15 @@ async function runSmokeTest(smokeTestDefn, testOptions) {
|
|
|
184
184
|
bufferedConsole.log(` Retrying run (${i} out of ${retries} retries)…`);
|
|
185
185
|
}
|
|
186
186
|
|
|
187
|
-
let
|
|
187
|
+
let config = smokeTestDefn.config;
|
|
188
188
|
if (useLegacyNavigation) {
|
|
189
|
-
|
|
189
|
+
config = convertToLegacyConfig(config);
|
|
190
190
|
}
|
|
191
191
|
|
|
192
192
|
// Run Lighthouse.
|
|
193
193
|
try {
|
|
194
194
|
result = {
|
|
195
|
-
...await lighthouseRunner(requestedUrl,
|
|
195
|
+
...await lighthouseRunner(requestedUrl, config, {isDebug, useLegacyNavigation}),
|
|
196
196
|
networkRequests: takeNetworkRequestUrls ? takeNetworkRequestUrls() : undefined,
|
|
197
197
|
};
|
|
198
198
|
|
|
@@ -10,6 +10,14 @@ import {Audit} from '../audit.js';
|
|
|
10
10
|
import {MainResource} from '../../computed/main-resource.js';
|
|
11
11
|
import * as i18n from '../../lib/i18n/i18n.js';
|
|
12
12
|
|
|
13
|
+
const BOT_USER_AGENTS = new Set([
|
|
14
|
+
undefined,
|
|
15
|
+
'Googlebot',
|
|
16
|
+
'bingbot',
|
|
17
|
+
'DuckDuckBot',
|
|
18
|
+
'archive.org_bot',
|
|
19
|
+
]);
|
|
20
|
+
|
|
13
21
|
const BLOCKLIST = new Set([
|
|
14
22
|
'noindex',
|
|
15
23
|
'none',
|
|
@@ -48,7 +56,7 @@ function isUnavailable(directive) {
|
|
|
48
56
|
|
|
49
57
|
/**
|
|
50
58
|
* Returns true if any of provided directives blocks page from being indexed
|
|
51
|
-
* @param {string} directives
|
|
59
|
+
* @param {string} directives assumes no user-agent prefix
|
|
52
60
|
* @return {boolean}
|
|
53
61
|
*/
|
|
54
62
|
function hasBlockingDirective(directives) {
|
|
@@ -58,16 +66,18 @@ function hasBlockingDirective(directives) {
|
|
|
58
66
|
}
|
|
59
67
|
|
|
60
68
|
/**
|
|
61
|
-
* Returns
|
|
69
|
+
* Returns user agent if specified in robots header (e.g. `googlebot: noindex`)
|
|
62
70
|
* @param {string} directives
|
|
63
|
-
* @return {
|
|
71
|
+
* @return {string|undefined}
|
|
64
72
|
*/
|
|
65
|
-
function
|
|
73
|
+
function getUserAgentFromHeaderDirectives(directives) {
|
|
66
74
|
const parts = directives.match(/^([^,:]+):/);
|
|
67
75
|
|
|
68
76
|
// Check if directives are prefixed with `googlebot:`, `googlebot-news:`, `otherbot:`, etc.
|
|
69
77
|
// but ignore `unavailable_after:` which is a valid directive
|
|
70
|
-
|
|
78
|
+
if (!!parts && parts[1].toLowerCase() !== UNAVAILABLE_AFTER) {
|
|
79
|
+
return parts[1];
|
|
80
|
+
}
|
|
71
81
|
}
|
|
72
82
|
|
|
73
83
|
class IsCrawlable extends Audit {
|
|
@@ -85,6 +95,74 @@ class IsCrawlable extends Audit {
|
|
|
85
95
|
};
|
|
86
96
|
}
|
|
87
97
|
|
|
98
|
+
/**
|
|
99
|
+
* @param {LH.Artifacts.MetaElement} metaElement
|
|
100
|
+
*/
|
|
101
|
+
static handleMetaElement(metaElement) {
|
|
102
|
+
const content = metaElement.content || '';
|
|
103
|
+
if (hasBlockingDirective(content)) {
|
|
104
|
+
return {
|
|
105
|
+
source: {
|
|
106
|
+
...Audit.makeNodeItem(metaElement.node),
|
|
107
|
+
snippet: `<meta name="${metaElement.name}" content="${content}" />`,
|
|
108
|
+
},
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* @param {string|undefined} userAgent
|
|
115
|
+
* @param {LH.Artifacts.NetworkRequest} mainResource
|
|
116
|
+
* @param {LH.Artifacts.MetaElement[]} metaElements
|
|
117
|
+
* @param {import('robots-parser').Robot|undefined} parsedRobotsTxt
|
|
118
|
+
* @param {URL} robotsTxtUrl
|
|
119
|
+
*/
|
|
120
|
+
static determineIfCrawlableForUserAgent(userAgent, mainResource, metaElements,
|
|
121
|
+
parsedRobotsTxt, robotsTxtUrl) {
|
|
122
|
+
/** @type {LH.Audit.Details.Table['items']} */
|
|
123
|
+
const blockingDirectives = [];
|
|
124
|
+
|
|
125
|
+
// Prefer a meta element specific to a user agent, fallback to generic 'robots' if not present.
|
|
126
|
+
// https://developers.google.com/search/blog/2007/03/using-robots-meta-tag#directing-a-robots-meta-tag-specifically-at-googlebot
|
|
127
|
+
let meta;
|
|
128
|
+
if (userAgent) meta = metaElements.find(meta => meta.name === userAgent.toLowerCase());
|
|
129
|
+
if (!meta) meta = metaElements.find(meta => meta.name === 'robots');
|
|
130
|
+
if (meta) {
|
|
131
|
+
const blockingDirective = IsCrawlable.handleMetaElement(meta);
|
|
132
|
+
if (blockingDirective) blockingDirectives.push(blockingDirective);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
for (const header of mainResource.responseHeaders || []) {
|
|
136
|
+
if (header.name.toLowerCase() !== ROBOTS_HEADER) continue;
|
|
137
|
+
|
|
138
|
+
const directiveUserAgent = getUserAgentFromHeaderDirectives(header.value);
|
|
139
|
+
if (directiveUserAgent !== userAgent && directiveUserAgent !== undefined) continue;
|
|
140
|
+
|
|
141
|
+
let directiveWithoutUserAgentPrefix = header.value.trim();
|
|
142
|
+
if (userAgent && header.value.startsWith(`${userAgent}:`)) {
|
|
143
|
+
directiveWithoutUserAgentPrefix = header.value.replace(`${userAgent}:`, '');
|
|
144
|
+
}
|
|
145
|
+
if (!hasBlockingDirective(directiveWithoutUserAgentPrefix)) continue;
|
|
146
|
+
|
|
147
|
+
blockingDirectives.push({source: `${header.name}: ${header.value}`});
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
if (parsedRobotsTxt && !parsedRobotsTxt.isAllowed(mainResource.url, userAgent)) {
|
|
151
|
+
const line = parsedRobotsTxt.getMatchingLineNumber(mainResource.url) || 1;
|
|
152
|
+
blockingDirectives.push({
|
|
153
|
+
source: {
|
|
154
|
+
type: /** @type {const} */ ('source-location'),
|
|
155
|
+
url: robotsTxtUrl.href,
|
|
156
|
+
urlProvider: /** @type {const} */ ('network'),
|
|
157
|
+
line: line - 1,
|
|
158
|
+
column: 0,
|
|
159
|
+
},
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
return blockingDirectives;
|
|
164
|
+
}
|
|
165
|
+
|
|
88
166
|
/**
|
|
89
167
|
* @param {LH.Artifacts} artifacts
|
|
90
168
|
* @param {LH.Audit.Context} context
|
|
@@ -92,57 +170,51 @@ class IsCrawlable extends Audit {
|
|
|
92
170
|
*/
|
|
93
171
|
static async audit(artifacts, context) {
|
|
94
172
|
const devtoolsLog = artifacts.devtoolsLogs[Audit.DEFAULT_PASS];
|
|
95
|
-
const metaRobots = artifacts.MetaElements.find(meta => meta.name === 'robots');
|
|
96
173
|
const mainResource = await MainResource.request({devtoolsLog, URL: artifacts.URL}, context);
|
|
97
|
-
/** @type {LH.Audit.Details.Table['items']} */
|
|
98
|
-
const blockingDirectives = [];
|
|
99
174
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
175
|
+
const robotsTxtUrl = new URL('/robots.txt', mainResource.url);
|
|
176
|
+
const parsedRobotsTxt = artifacts.RobotsTxt.content ?
|
|
177
|
+
robotsParser(robotsTxtUrl.href, artifacts.RobotsTxt.content) :
|
|
178
|
+
undefined;
|
|
179
|
+
|
|
180
|
+
// Only fail if all known bots and generic bots (UserAgent '*' or 'robots' directive)
|
|
181
|
+
// are blocked from crawling.
|
|
182
|
+
// If at least one bot is allowed, we pass the audit. Any known bots that are not allowed
|
|
183
|
+
// will be listed in a warning.
|
|
184
|
+
|
|
185
|
+
/** @type {Array<string|undefined>} */
|
|
186
|
+
const blockedUserAgents = [];
|
|
187
|
+
const genericBlockingDirectives = [];
|
|
188
|
+
|
|
189
|
+
for (const userAgent of BOT_USER_AGENTS) {
|
|
190
|
+
const blockingDirectives = IsCrawlable.determineIfCrawlableForUserAgent(
|
|
191
|
+
userAgent, mainResource, artifacts.MetaElements, parsedRobotsTxt, robotsTxtUrl);
|
|
192
|
+
if (blockingDirectives.length > 0) {
|
|
193
|
+
blockedUserAgents.push(userAgent);
|
|
194
|
+
}
|
|
195
|
+
if (userAgent === undefined) {
|
|
196
|
+
genericBlockingDirectives.push(...blockingDirectives);
|
|
111
197
|
}
|
|
112
198
|
}
|
|
113
199
|
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
const robotsFileUrl = new URL('/robots.txt', mainResource.url);
|
|
121
|
-
const robotsTxt = robotsParser(robotsFileUrl.href, artifacts.RobotsTxt.content);
|
|
122
|
-
|
|
123
|
-
if (!robotsTxt.isAllowed(mainResource.url)) {
|
|
124
|
-
const line = robotsTxt.getMatchingLineNumber(mainResource.url) || 1;
|
|
125
|
-
blockingDirectives.push({
|
|
126
|
-
source: {
|
|
127
|
-
type: /** @type {const} */ ('source-location'),
|
|
128
|
-
url: robotsFileUrl.href,
|
|
129
|
-
urlProvider: /** @type {const} */ ('network'),
|
|
130
|
-
line: line - 1,
|
|
131
|
-
column: 0,
|
|
132
|
-
},
|
|
133
|
-
});
|
|
134
|
-
}
|
|
200
|
+
const score = blockedUserAgents.length === BOT_USER_AGENTS.size ? 0 : 1;
|
|
201
|
+
const warnings = [];
|
|
202
|
+
if (score && blockedUserAgents.length > 0) {
|
|
203
|
+
const list = blockedUserAgents.filter(Boolean).join(', ');
|
|
204
|
+
// eslint-disable-next-line max-len
|
|
205
|
+
warnings.push(`The following bot user agents are blocked from crawling: ${list}. The audit is otherwise passing, because at least one bot was explicitly allowed.`);
|
|
135
206
|
}
|
|
136
207
|
|
|
137
208
|
/** @type {LH.Audit.Details.Table['headings']} */
|
|
138
209
|
const headings = [
|
|
139
210
|
{key: 'source', valueType: 'code', label: 'Blocking Directive Source'},
|
|
140
211
|
];
|
|
141
|
-
const details = Audit.makeTableDetails(headings,
|
|
212
|
+
const details = Audit.makeTableDetails(headings, score === 0 ? genericBlockingDirectives : []);
|
|
142
213
|
|
|
143
214
|
return {
|
|
144
|
-
score
|
|
215
|
+
score,
|
|
145
216
|
details,
|
|
217
|
+
warnings,
|
|
146
218
|
};
|
|
147
219
|
}
|
|
148
220
|
}
|
|
@@ -364,18 +364,18 @@ function resolveSettings(settingsJson = {}, overrides = undefined) {
|
|
|
364
364
|
}
|
|
365
365
|
|
|
366
366
|
/**
|
|
367
|
-
* @param {LH.Config
|
|
367
|
+
* @param {LH.Config} config
|
|
368
368
|
* @param {string | undefined} configDir
|
|
369
369
|
* @param {{plugins?: string[]} | undefined} flags
|
|
370
|
-
* @return {Promise<LH.Config
|
|
370
|
+
* @return {Promise<LH.Config>}
|
|
371
371
|
*/
|
|
372
|
-
async function mergePlugins(
|
|
373
|
-
const configPlugins =
|
|
372
|
+
async function mergePlugins(config, configDir, flags) {
|
|
373
|
+
const configPlugins = config.plugins || [];
|
|
374
374
|
const flagPlugins = flags?.plugins || [];
|
|
375
375
|
const pluginNames = new Set([...configPlugins, ...flagPlugins]);
|
|
376
376
|
|
|
377
377
|
for (const pluginName of pluginNames) {
|
|
378
|
-
validation.assertValidPluginName(
|
|
378
|
+
validation.assertValidPluginName(config, pluginName);
|
|
379
379
|
|
|
380
380
|
// In bundled contexts, `resolveModulePath` will fail, so use the raw pluginName directly.
|
|
381
381
|
const pluginPath = isBundledEnvironment() ?
|
|
@@ -384,10 +384,10 @@ async function mergePlugins(configJSON, configDir, flags) {
|
|
|
384
384
|
const rawPluginJson = await requireWrapper(pluginPath);
|
|
385
385
|
const pluginJson = ConfigPlugin.parsePlugin(rawPluginJson, pluginName);
|
|
386
386
|
|
|
387
|
-
|
|
387
|
+
config = mergeConfigFragment(config, pluginJson);
|
|
388
388
|
}
|
|
389
389
|
|
|
390
|
-
return
|
|
390
|
+
return config;
|
|
391
391
|
}
|
|
392
392
|
|
|
393
393
|
|
|
@@ -428,7 +428,7 @@ async function resolveGathererToDefn(gathererJson, coreGathererList, configDir)
|
|
|
428
428
|
* Take an array of audits and audit paths and require any paths (possibly
|
|
429
429
|
* relative to the optional `configDir`) using `resolveModule`,
|
|
430
430
|
* leaving only an array of AuditDefns.
|
|
431
|
-
* @param {LH.Config
|
|
431
|
+
* @param {LH.Config['audits']} audits
|
|
432
432
|
* @param {string=} configDir
|
|
433
433
|
* @return {Promise<Array<LH.Config.AuditDefn>|null>}
|
|
434
434
|
*/
|
|
@@ -585,10 +585,10 @@ function deepClone(json) {
|
|
|
585
585
|
}
|
|
586
586
|
|
|
587
587
|
/**
|
|
588
|
-
* Deep clone a
|
|
588
|
+
* Deep clone a config, copying over any "live" gatherer or audit that
|
|
589
589
|
* wouldn't make the JSON round trip.
|
|
590
|
-
* @param {LH.Config
|
|
591
|
-
* @return {LH.Config
|
|
590
|
+
* @param {LH.Config} json
|
|
591
|
+
* @return {LH.Config}
|
|
592
592
|
*/
|
|
593
593
|
function deepCloneConfigJson(json) {
|
|
594
594
|
const cloned = deepClone(json);
|
|
@@ -215,11 +215,11 @@ class ConfigPlugin {
|
|
|
215
215
|
}
|
|
216
216
|
|
|
217
217
|
/**
|
|
218
|
-
* Extracts and validates a
|
|
218
|
+
* Extracts and validates a config from the provided plugin input, throwing
|
|
219
219
|
* if it deviates from the expected object shape.
|
|
220
220
|
* @param {unknown} pluginJson
|
|
221
221
|
* @param {string} pluginName
|
|
222
|
-
* @return {LH.Config
|
|
222
|
+
* @return {LH.Config}
|
|
223
223
|
*/
|
|
224
224
|
static parsePlugin(pluginJson, pluginName) {
|
|
225
225
|
// Clone to prevent modifications of original and to deactivate any live properties.
|
package/core/config/config.js
CHANGED
|
@@ -38,19 +38,19 @@ const defaultConfigPath = path.join(
|
|
|
38
38
|
);
|
|
39
39
|
|
|
40
40
|
/**
|
|
41
|
-
* @param {LH.Config
|
|
41
|
+
* @param {LH.Config|undefined} config
|
|
42
42
|
* @param {{configPath?: string}} context
|
|
43
|
-
* @return {{configWorkingCopy: LH.Config
|
|
43
|
+
* @return {{configWorkingCopy: LH.Config, configDir?: string, configPath?: string}}
|
|
44
44
|
*/
|
|
45
|
-
function resolveWorkingCopy(
|
|
45
|
+
function resolveWorkingCopy(config, context) {
|
|
46
46
|
let {configPath} = context;
|
|
47
47
|
|
|
48
48
|
if (configPath && !path.isAbsolute(configPath)) {
|
|
49
49
|
throw new Error('configPath must be an absolute path');
|
|
50
50
|
}
|
|
51
51
|
|
|
52
|
-
if (!
|
|
53
|
-
|
|
52
|
+
if (!config) {
|
|
53
|
+
config = defaultConfig;
|
|
54
54
|
configPath = defaultConfigPath;
|
|
55
55
|
}
|
|
56
56
|
|
|
@@ -58,24 +58,24 @@ function resolveWorkingCopy(configJSON, context) {
|
|
|
58
58
|
const configDir = configPath ? path.dirname(configPath) : undefined;
|
|
59
59
|
|
|
60
60
|
return {
|
|
61
|
-
configWorkingCopy: deepCloneConfigJson(
|
|
61
|
+
configWorkingCopy: deepCloneConfigJson(config),
|
|
62
62
|
configPath,
|
|
63
63
|
configDir,
|
|
64
64
|
};
|
|
65
65
|
}
|
|
66
66
|
|
|
67
67
|
/**
|
|
68
|
-
* @param {LH.Config
|
|
69
|
-
* @return {LH.Config
|
|
68
|
+
* @param {LH.Config} config
|
|
69
|
+
* @return {LH.Config}
|
|
70
70
|
*/
|
|
71
|
-
function resolveExtensions(
|
|
72
|
-
if (!
|
|
71
|
+
function resolveExtensions(config) {
|
|
72
|
+
if (!config.extends) return config;
|
|
73
73
|
|
|
74
|
-
if (
|
|
74
|
+
if (config.extends !== 'lighthouse:default') {
|
|
75
75
|
throw new Error('`lighthouse:default` is the only valid extension method.');
|
|
76
76
|
}
|
|
77
77
|
|
|
78
|
-
const {artifacts, ...extensionJSON} =
|
|
78
|
+
const {artifacts, ...extensionJSON} = config;
|
|
79
79
|
const defaultClone = deepCloneConfigJson(defaultConfig);
|
|
80
80
|
const mergedConfig = mergeConfigFragment(defaultClone, extensionJSON);
|
|
81
81
|
|
|
@@ -236,15 +236,15 @@ function resolveFakeNavigations(artifactDefns, settings) {
|
|
|
236
236
|
|
|
237
237
|
/**
|
|
238
238
|
* @param {LH.Gatherer.GatherMode} gatherMode
|
|
239
|
-
* @param {LH.Config
|
|
239
|
+
* @param {LH.Config=} config
|
|
240
240
|
* @param {LH.Flags=} flags
|
|
241
241
|
* @return {Promise<{resolvedConfig: LH.Config.ResolvedConfig, warnings: string[]}>}
|
|
242
242
|
*/
|
|
243
|
-
async function initializeConfig(gatherMode,
|
|
243
|
+
async function initializeConfig(gatherMode, config, flags = {}) {
|
|
244
244
|
const status = {msg: 'Initialize config', id: 'lh:config'};
|
|
245
245
|
log.time(status, 'verbose');
|
|
246
246
|
|
|
247
|
-
let {configWorkingCopy, configDir} = resolveWorkingCopy(
|
|
247
|
+
let {configWorkingCopy, configDir} = resolveWorkingCopy(config, flags);
|
|
248
248
|
|
|
249
249
|
configWorkingCopy = resolveExtensions(configWorkingCopy);
|
|
250
250
|
configWorkingCopy = await mergePlugins(configWorkingCopy, configDir, flags);
|
|
@@ -170,7 +170,7 @@ for (const key of Object.keys(artifacts)) {
|
|
|
170
170
|
artifacts[/** @type {keyof typeof artifacts} */ (key)] = key;
|
|
171
171
|
}
|
|
172
172
|
|
|
173
|
-
/** @type {LH.Config
|
|
173
|
+
/** @type {LH.Config} */
|
|
174
174
|
const defaultConfig = {
|
|
175
175
|
settings: constants.defaultSettings,
|
|
176
176
|
artifacts: [
|
|
@@ -517,7 +517,6 @@ const defaultConfig = {
|
|
|
517
517
|
{id: 'non-composited-animations', weight: 0},
|
|
518
518
|
{id: 'unsized-images', weight: 0},
|
|
519
519
|
{id: 'viewport', weight: 0},
|
|
520
|
-
{id: 'no-unload-listeners', weight: 0},
|
|
521
520
|
{id: 'uses-responsive-images-snapshot', weight: 0},
|
|
522
521
|
{id: 'work-during-interaction', weight: 0},
|
|
523
522
|
{id: 'bf-cache', weight: 0},
|
|
@@ -623,6 +622,7 @@ const defaultConfig = {
|
|
|
623
622
|
{id: 'doctype', weight: 1, group: 'best-practices-browser-compat'},
|
|
624
623
|
{id: 'charset', weight: 1, group: 'best-practices-browser-compat'},
|
|
625
624
|
// General Group
|
|
625
|
+
{id: 'no-unload-listeners', weight: 1, group: 'best-practices-general'},
|
|
626
626
|
{id: 'js-libraries', weight: 0, group: 'best-practices-general'},
|
|
627
627
|
{id: 'deprecations', weight: 1, group: 'best-practices-general'},
|
|
628
628
|
{id: 'errors-in-console', weight: 1, group: 'best-practices-general'},
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
-
/** @type {LH.Config
|
|
7
|
+
/** @type {LH.Config} */
|
|
8
8
|
const fullConfig = {
|
|
9
9
|
extends: 'lighthouse:default',
|
|
10
10
|
settings: {},
|