@mui/internal-code-infra 0.0.4-canary.9 → 0.0.4-canary.91

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (140) hide show
  1. package/LICENSE +1 -1
  2. package/README.md +19 -8
  3. package/build/babel-config.d.mts +5 -1
  4. package/build/brokenLinksChecker/crawlWorker.d.mts +1 -0
  5. package/build/brokenLinksChecker/index.d.mts +45 -2
  6. package/build/changelog/types.d.ts +3 -1
  7. package/build/cli/cmdArgosPush.d.mts +2 -2
  8. package/build/cli/cmdBuild.d.mts +3 -2
  9. package/build/cli/cmdCopyFiles.d.mts +2 -2
  10. package/build/cli/cmdExtractErrorCodes.d.mts +2 -2
  11. package/build/cli/cmdGenerateChangelog.d.mts +2 -2
  12. package/build/cli/cmdGithubAuth.d.mts +2 -2
  13. package/build/cli/cmdListWorkspaces.d.mts +6 -4
  14. package/build/cli/cmdNetlifyIgnore.d.mts +3 -2
  15. package/build/cli/cmdPublish.d.mts +4 -2
  16. package/build/cli/cmdPublishCanary.d.mts +3 -3
  17. package/build/cli/cmdPublishNewPackage.d.mts +4 -2
  18. package/build/cli/cmdSetVersionOverrides.d.mts +2 -2
  19. package/build/cli/cmdVale.d.mts +46 -0
  20. package/build/cli/cmdValidateBuiltTypes.d.mts +2 -2
  21. package/build/eslint/baseConfig.d.mts +3 -1
  22. package/build/eslint/mui/rules/disallow-react-api-in-server-components.d.mts +2 -2
  23. package/build/eslint/mui/rules/docgen-ignore-before-comment.d.mts +2 -2
  24. package/build/eslint/mui/rules/no-floating-cleanup.d.mts +5 -0
  25. package/build/eslint/mui/rules/no-guarded-throw.d.mts +31 -0
  26. package/build/eslint/mui/rules/no-presentation-role.d.mts +5 -0
  27. package/build/eslint/mui/rules/no-restricted-resolved-imports.d.mts +2 -2
  28. package/build/eslint/mui/rules/nodeEnvUtils.d.mts +18 -0
  29. package/build/remark/config.d.mts +43 -0
  30. package/build/remark/createLintTester.d.mts +10 -0
  31. package/build/remark/firstBlockHeading.d.mts +4 -0
  32. package/build/remark/gitDiff.d.mts +2 -0
  33. package/build/remark/noSpaceInLinks.d.mts +2 -0
  34. package/build/remark/straightQuotes.d.mts +2 -0
  35. package/build/remark/tableAlignment.d.mts +2 -0
  36. package/build/remark/terminalLanguage.d.mts +2 -0
  37. package/build/utils/babel.d.mts +1 -1
  38. package/build/utils/build.d.mts +56 -37
  39. package/build/utils/git.d.mts +7 -0
  40. package/build/utils/github.d.mts +1 -1
  41. package/build/utils/pnpm.d.mts +81 -2
  42. package/build/utils/testUtils.d.mts +7 -0
  43. package/build/utils/typescript.d.mts +6 -16
  44. package/package.json +74 -50
  45. package/src/babel-config.mjs +3 -1
  46. package/src/brokenLinksChecker/crawlWorker.mjs +240 -0
  47. package/src/brokenLinksChecker/index.mjs +263 -188
  48. package/src/changelog/fetchChangelogs.mjs +8 -2
  49. package/src/changelog/renderChangelog.mjs +1 -1
  50. package/src/changelog/types.ts +3 -1
  51. package/src/cli/cmdBuild.mjs +51 -85
  52. package/src/cli/cmdListWorkspaces.mjs +12 -27
  53. package/src/cli/cmdNetlifyIgnore.mjs +34 -93
  54. package/src/cli/cmdPublish.mjs +95 -20
  55. package/src/cli/cmdPublishCanary.mjs +128 -132
  56. package/src/cli/cmdPublishNewPackage.mjs +27 -6
  57. package/src/cli/cmdSetVersionOverrides.mjs +8 -9
  58. package/src/cli/cmdVale.mjs +513 -0
  59. package/src/cli/index.mjs +2 -0
  60. package/src/cli/packageJson.d.ts +2 -3
  61. package/src/eslint/baseConfig.mjs +51 -19
  62. package/src/eslint/docsConfig.mjs +2 -1
  63. package/src/eslint/mui/config.mjs +24 -4
  64. package/src/eslint/mui/index.mjs +6 -0
  65. package/src/eslint/mui/rules/no-floating-cleanup.mjs +187 -0
  66. package/src/eslint/mui/rules/no-guarded-throw.mjs +115 -0
  67. package/src/eslint/mui/rules/no-presentation-role.mjs +60 -0
  68. package/src/eslint/mui/rules/nodeEnvUtils.mjs +52 -0
  69. package/src/eslint/mui/rules/require-dev-wrapper.mjs +25 -40
  70. package/src/estree-typescript.d.ts +1 -1
  71. package/src/remark/config.mjs +157 -0
  72. package/src/remark/createLintTester.mjs +19 -0
  73. package/src/remark/firstBlockHeading.mjs +87 -0
  74. package/src/remark/gitDiff.mjs +43 -0
  75. package/src/remark/noSpaceInLinks.mjs +42 -0
  76. package/src/remark/straightQuotes.mjs +31 -0
  77. package/src/remark/tableAlignment.mjs +23 -0
  78. package/src/remark/terminalLanguage.mjs +19 -0
  79. package/src/untyped-plugins.d.ts +11 -11
  80. package/src/utils/build.mjs +604 -270
  81. package/src/utils/git.mjs +27 -7
  82. package/src/utils/pnpm.mjs +277 -10
  83. package/src/utils/testUtils.mjs +18 -0
  84. package/src/utils/typescript.mjs +23 -42
  85. package/vale/.vale.ini +1 -0
  86. package/vale/styles/MUI/CorrectReferenceAllCases.yml +42 -0
  87. package/vale/styles/MUI/CorrectRererenceCased.yml +14 -0
  88. package/vale/styles/MUI/GoogleLatin.yml +11 -0
  89. package/vale/styles/MUI/MuiBrandName.yml +22 -0
  90. package/vale/styles/MUI/NoBritish.yml +112 -0
  91. package/vale/styles/MUI/NoCompanyName.yml +17 -0
  92. package/build/markdownlint/duplicate-h1.d.mts +0 -56
  93. package/build/markdownlint/git-diff.d.mts +0 -11
  94. package/build/markdownlint/index.d.mts +0 -52
  95. package/build/markdownlint/straight-quotes.d.mts +0 -11
  96. package/build/markdownlint/table-alignment.d.mts +0 -11
  97. package/build/markdownlint/terminal-language.d.mts +0 -11
  98. package/src/brokenLinksChecker/__fixtures__/static-site/broken-links.html +0 -20
  99. package/src/brokenLinksChecker/__fixtures__/static-site/broken-targets.html +0 -22
  100. package/src/brokenLinksChecker/__fixtures__/static-site/example.md +0 -20
  101. package/src/brokenLinksChecker/__fixtures__/static-site/external-links.html +0 -21
  102. package/src/brokenLinksChecker/__fixtures__/static-site/ignored-page.html +0 -17
  103. package/src/brokenLinksChecker/__fixtures__/static-site/index.html +0 -28
  104. package/src/brokenLinksChecker/__fixtures__/static-site/known-targets.json +0 -5
  105. package/src/brokenLinksChecker/__fixtures__/static-site/nested/page.html +0 -21
  106. package/src/brokenLinksChecker/__fixtures__/static-site/orphaned-page.html +0 -20
  107. package/src/brokenLinksChecker/__fixtures__/static-site/page-with-api-links.html +0 -20
  108. package/src/brokenLinksChecker/__fixtures__/static-site/page-with-custom-targets.html +0 -24
  109. package/src/brokenLinksChecker/__fixtures__/static-site/page-with-ignored-content.html +0 -28
  110. package/src/brokenLinksChecker/__fixtures__/static-site/page-with-known-target-links.html +0 -19
  111. package/src/brokenLinksChecker/__fixtures__/static-site/unclosed-tags.html +0 -1
  112. package/src/brokenLinksChecker/__fixtures__/static-site/valid.html +0 -20
  113. package/src/brokenLinksChecker/__fixtures__/static-site/with-anchors.html +0 -31
  114. package/src/brokenLinksChecker/index.test.ts +0 -261
  115. package/src/changelog/categorizeCommits.test.ts +0 -319
  116. package/src/changelog/filterCommits.test.ts +0 -363
  117. package/src/changelog/parseCommitLabels.test.ts +0 -509
  118. package/src/changelog/renderChangelog.test.ts +0 -378
  119. package/src/changelog/sortSections.test.ts +0 -199
  120. package/src/eslint/mui/rules/add-undef-to-optional.test.mjs +0 -361
  121. package/src/eslint/mui/rules/consistent-production-guard.test.mjs +0 -162
  122. package/src/eslint/mui/rules/disallow-active-elements-as-key-event-target.test.mjs +0 -66
  123. package/src/eslint/mui/rules/disallow-react-api-in-server-components.test.mjs +0 -305
  124. package/src/eslint/mui/rules/docgen-ignore-before-comment.test.mjs +0 -52
  125. package/src/eslint/mui/rules/flatten-parentheses.test.mjs +0 -245
  126. package/src/eslint/mui/rules/mui-name-matches-component-name.test.mjs +0 -247
  127. package/src/eslint/mui/rules/no-empty-box.test.mjs +0 -40
  128. package/src/eslint/mui/rules/no-styled-box.test.mjs +0 -73
  129. package/src/eslint/mui/rules/require-dev-wrapper.test.mjs +0 -265
  130. package/src/eslint/mui/rules/rules-of-use-theme-variants.test.mjs +0 -149
  131. package/src/eslint/mui/rules/straight-quotes.test.mjs +0 -67
  132. package/src/markdownlint/duplicate-h1.mjs +0 -69
  133. package/src/markdownlint/git-diff.mjs +0 -31
  134. package/src/markdownlint/index.mjs +0 -62
  135. package/src/markdownlint/straight-quotes.mjs +0 -26
  136. package/src/markdownlint/table-alignment.mjs +0 -42
  137. package/src/markdownlint/terminal-language.mjs +0 -19
  138. package/src/utils/build.test.mjs +0 -705
  139. package/src/utils/template.test.mjs +0 -133
  140. package/src/utils/typescript.test.mjs +0 -380
@@ -1,20 +1,17 @@
1
1
  /* eslint-disable no-console */
2
2
  import { execaCommand } from 'execa';
3
3
  import timers from 'node:timers/promises';
4
- import { parse } from 'node-html-parser';
5
4
  import * as fs from 'node:fs/promises';
6
5
  import * as path from 'node:path';
6
+ import { pathToFileURL } from 'node:url';
7
7
  import chalk from 'chalk';
8
8
  import { Transform } from 'node:stream';
9
- import contentType from 'content-type';
10
- import { unified } from 'unified';
11
- import remarkParse from 'remark-parse';
12
- import remarkGfm from 'remark-gfm';
13
- import remarkRehype from 'remark-rehype';
14
- import rehypeSlug from 'rehype-slug';
15
- import rehypeStringify from 'rehype-stringify';
9
+ import { Worker } from 'node:worker_threads';
16
10
 
17
11
  const DEFAULT_CONCURRENCY = 4;
12
+ const SERVER_START_TIMEOUT = 10000;
13
+
14
+ const crawlWorkerUrl = new URL('./crawlWorker.mjs', import.meta.url);
18
15
 
19
16
  /**
20
17
  * Creates a Transform stream that prefixes each line with a given string.
@@ -105,6 +102,30 @@ function deserializeLinkStructure(data) {
105
102
  return linkStructure;
106
103
  }
107
104
 
105
+ /**
106
+ * Input data passed to the crawl worker via workerData.
107
+ * @typedef {Object} CrawlWorkerInput
108
+ * @property {string} pageUrl - The page URL to crawl
109
+ * @property {ResolvedCrawlOptions} options - Fully resolved crawl options
110
+ */
111
+
112
+ /**
113
+ * Serialized page data returned by the crawl worker (uses arrays instead of Sets for structured clone).
114
+ * @typedef {Object} CrawlWorkerPageData
115
+ * @property {string} url - The normalized page URL
116
+ * @property {number} status - HTTP status code
117
+ * @property {string[]} targets - Array of anchor targets (e.g., '#intro')
118
+ * @property {string} contentType - Content-type of the page
119
+ */
120
+
121
+ /**
122
+ * Output message posted by the crawl worker.
123
+ * @typedef {Object} CrawlWorkerOutput
124
+ * @property {CrawlWorkerPageData} pageData - Serialized page data
125
+ * @property {Link[]} links - Links discovered on the page
126
+ * @property {{ pageUrl: string, results: import('html-validate').Result[] } | null} htmlValidateResults - HTML validation results, or null if validation was skipped/passed
127
+ */
128
+
108
129
  /**
109
130
  * Data about a crawled page including its URL, HTTP status, and available link targets.
110
131
  * @typedef {Object} PageData
@@ -131,77 +152,6 @@ async function writePagesToFile(pages, outPath) {
131
152
  await fs.writeFile(outPath, JSON.stringify(fileContent, null, 2), 'utf-8');
132
153
  }
133
154
 
134
- /**
135
- * Computes the accessible name of an element according to ARIA rules.
136
- * Polyfill for `node.computedName` available only in Chrome v112+.
137
- * Checks in order: aria-label, aria-labelledby, label[for], img alt, innerText.
138
- * @param {import('node-html-parser').HTMLElement | null} elm - Element to compute name for
139
- * @param {import('node-html-parser').HTMLElement} ownerDocument - Document containing the element
140
- * @returns {string} The computed accessible name, or empty string if none found
141
- */
142
- function getAccessibleName(elm, ownerDocument) {
143
- if (!elm) {
144
- return '';
145
- }
146
-
147
- // 1. aria-label
148
- const ariaLabel = elm.getAttribute('aria-label')?.trim();
149
- if (ariaLabel) {
150
- return ariaLabel;
151
- }
152
-
153
- // 2. aria-labelledby
154
- const labelledby = elm.getAttribute('aria-labelledby');
155
- if (labelledby) {
156
- const labels = [];
157
- for (const id of labelledby.split(/\s+/)) {
158
- const label = getAccessibleName(ownerDocument.getElementById(id), ownerDocument);
159
- if (label) {
160
- labels.push(label);
161
- }
162
- }
163
- const label = labels.join(' ').trim();
164
- if (label) {
165
- return label;
166
- }
167
- }
168
-
169
- // 3. <label for="id">
170
- if (elm.id) {
171
- const label = ownerDocument.querySelector(`label[for="${elm.id}"]`);
172
- if (label) {
173
- return getAccessibleName(label, ownerDocument);
174
- }
175
- }
176
-
177
- // 4. <img alt="">
178
- if (elm.tagName === 'IMG') {
179
- const alt = elm.getAttribute('alt')?.trim();
180
- if (alt) {
181
- return alt;
182
- }
183
- }
184
-
185
- // 5. Fallback: visible text
186
- return elm.innerText.trim();
187
- }
188
-
189
- /**
190
- * Converts markdown content to HTML using unified pipeline.
191
- * @param {string} markdown - Raw markdown content
192
- * @returns {Promise<string>} Converted HTML string
193
- */
194
- async function markdownToHtml(markdown) {
195
- const result = await unified()
196
- .use(remarkParse)
197
- .use(remarkGfm)
198
- .use(remarkRehype)
199
- .use(rehypeSlug)
200
- .use(rehypeStringify)
201
- .process(markdown);
202
- return String(result);
203
- }
204
-
205
155
  /**
206
156
  * Generic concurrent task queue with configurable concurrency limit.
207
157
  * Processes tasks in FIFO order with a maximum number of concurrent workers.
@@ -402,11 +352,30 @@ function shouldIgnoreLink(link, ignores) {
402
352
  * @property {number} [concurrency] - Number of concurrent page fetches (defaults to 4)
403
353
  * @property {string[]} [seedUrls] - Starting URLs for the crawl (defaults to ['/'])
404
354
  * @property {IgnoreRule[]} [ignores] - Rules to ignore broken links. Each rule can have path, href, contentType, and/or has properties. All specified properties must match (AND logic). Within a property, multiple values use OR logic.
355
+ * @property {HtmlValidateOption} [htmlValidate] - Enable HTML validation on crawled pages. `false` (default): disabled. `true`: validate with recommended rules. Object: use as html-validate config — `mui:recommended` is always applied as the baseline, so most callers only need to set `rules`. Array: per-path config overrides — `mui:recommended` is applied once as the baseline and every entry whose `path` matches the page URL is layered on top; later matching entries win on conflicting rule keys. If an entry omits `extends`, it behaves like a rule patch and typically only changes the rules it names. If an entry includes `extends` (for example, re-extending `mui:recommended`), it can re-introduce or reset baseline presets rather than acting as a pure patch. An entry without `path` matches every page. If no entry matches, the page is not validated.
356
+ * @property {boolean} [verbose] - Log extra diagnostics during crawling (e.g. resolved html-validate config per page). Defaults to `false`.
357
+ */
358
+
359
+ /**
360
+ * Per-page HTML validation override entry.
361
+ * @typedef {Object} HtmlValidateOverride
362
+ * @property {(string | RegExp) | (string | RegExp)[]} [path] - Pattern(s) to match the page URL. Strings use exact match. Omit to match every page.
363
+ * @property {true | import('html-validate').ConfigData} config - html-validate config (or `true` for `mui:recommended`).
364
+ */
365
+
366
+ /**
367
+ * Public shape of the htmlValidate option.
368
+ * @typedef {boolean | import('html-validate').ConfigData | HtmlValidateOverride[]} HtmlValidateOption
369
+ */
370
+
371
+ /**
372
+ * Resolved per-page HTML validation entry. Empty array means validation is disabled.
373
+ * @typedef {{ path: (string | RegExp)[] | undefined, config: import('html-validate').ConfigData }} ResolvedHtmlValidateEntry
405
374
  */
406
375
 
407
376
  /**
408
377
  * Fully resolved configuration with all optional fields filled with defaults.
409
- * @typedef {Omit<Required<CrawlOptions>, 'ignores'> & { ignores: NormalizedIgnoreRule[] }} ResolvedCrawlOptions
378
+ * @typedef {Omit<Required<CrawlOptions>, 'ignores' | 'htmlValidate'> & { ignores: NormalizedIgnoreRule[], htmlValidate: ResolvedHtmlValidateEntry[] }} ResolvedCrawlOptions
410
379
  */
411
380
 
412
381
  /**
@@ -422,6 +391,42 @@ function validateIgnoreRule(rule) {
422
391
  }
423
392
  }
424
393
 
394
+ /**
395
+ * Normalizes a single config value to a non-null html-validate config object.
396
+ * Each config is registered as a pure rule patch; `mui:recommended` is pulled
397
+ * in once by the page's root config (ahead of every patch), so callers only
398
+ * need to specify the `rules` they want to change and never restate the
399
+ * recommended ruleset. `true` means "recommended only" (an empty patch). An
400
+ * explicit `extends` is still honored if a caller wants extra presets.
401
+ * @param {true | import('html-validate').ConfigData} config
402
+ * @returns {import('html-validate').ConfigData}
403
+ */
404
+ function normalizeHtmlValidateConfig(config) {
405
+ if (config === true) {
406
+ return {};
407
+ }
408
+ return config;
409
+ }
410
+
411
+ /**
412
+ * Resolves the htmlValidate option into an array of per-page entries.
413
+ * An empty array means validation is disabled.
414
+ * @param {HtmlValidateOption | undefined} option
415
+ * @returns {ResolvedHtmlValidateEntry[]}
416
+ */
417
+ function resolveHtmlValidateConfig(option) {
418
+ if (!option) {
419
+ return [];
420
+ }
421
+ if (option === true || !Array.isArray(option)) {
422
+ return [{ path: undefined, config: normalizeHtmlValidateConfig(option) }];
423
+ }
424
+ return option.map((entry) => ({
425
+ path: normalizeToArray(entry.path),
426
+ config: normalizeHtmlValidateConfig(entry.config),
427
+ }));
428
+ }
429
+
425
430
  /**
426
431
  * Resolves partial crawl options by filling in defaults for all optional fields.
427
432
  * @param {CrawlOptions} rawOptions - Partial options from user
@@ -447,6 +452,8 @@ function resolveOptions(rawOptions) {
447
452
  concurrency: rawOptions.concurrency ?? DEFAULT_CONCURRENCY,
448
453
  seedUrls: rawOptions.seedUrls ?? ['/'],
449
454
  ignores: normalizedIgnores,
455
+ htmlValidate: resolveHtmlValidateConfig(rawOptions.htmlValidate),
456
+ verbose: rawOptions.verbose ?? false,
450
457
  };
451
458
  }
452
459
 
@@ -506,25 +513,42 @@ async function resolveKnownTargets(options) {
506
513
 
507
514
  /**
508
515
  * Represents a broken link or broken link target discovered during crawling.
509
- * @typedef {Object} Issue
516
+ * @typedef {Object} BrokenLinkIssue
510
517
  * @property {'broken-link' | 'broken-target'} type - Type of issue: 'broken-link' for 404 pages, 'broken-target' for missing anchors
511
518
  * @property {string} message - Human-readable description of the issue (e.g., 'Target not found', 'Page returned error 404')
512
519
  * @property {Link} link - The link object that has the issue
513
520
  */
514
521
 
522
+ /**
523
+ * Represents an HTML validation issue found on a crawled page.
524
+ * @typedef {Object} HtmlValidateIssue
525
+ * @property {'html-validate'} type - Issue type discriminator
526
+ * @property {string} message - Human-readable description of the issue
527
+ * @property {string} pageUrl - The page URL where the issue was found
528
+ * @property {string} ruleId - The html-validate rule that triggered this issue (e.g., 'no-dup-id')
529
+ * @property {number} severity - Severity level (1 = warning, 2 = error)
530
+ * @property {{ line: number, column: number }} location - Source location of the issue
531
+ * @property {string | null} selector - DOM selector for the element, or null
532
+ */
533
+
534
+ /**
535
+ * Any issue discovered during crawling.
536
+ * @typedef {BrokenLinkIssue | HtmlValidateIssue} Issue
537
+ */
538
+
515
539
  /**
516
540
  * Results from a complete crawl operation.
517
541
  * @typedef {Object} CrawlResult
518
542
  * @property {Set<Link>} links - All links discovered during the crawl
519
543
  * @property {Map<string, PageData>} pages - All pages crawled, keyed by normalized URL
520
- * @property {Issue[]} issues - All broken links and broken targets found
544
+ * @property {Issue[]} issues - All issues found (broken links, broken targets, and HTML validation issues)
521
545
  */
522
546
 
523
547
  /**
524
548
  * Reports broken links to stderr, grouped by source page for better readability.
525
- * @param {Issue[]} issuesList - Array of issues to report
549
+ * @param {BrokenLinkIssue[]} issuesList - Array of broken link issues to report
526
550
  */
527
- function reportIssues(issuesList) {
551
+ function reportBrokenLinks(issuesList) {
528
552
  if (issuesList.length === 0) {
529
553
  return;
530
554
  }
@@ -532,7 +556,7 @@ function reportIssues(issuesList) {
532
556
  console.error('\nBroken links found:\n');
533
557
 
534
558
  // Group issues by source URL
535
- /** @type {Map<string, Issue[]>} */
559
+ /** @type {Map<string, BrokenLinkIssue[]>} */
536
560
  const issuesBySource = new Map();
537
561
  for (const issue of issuesList) {
538
562
  const sourceUrl = issue.link.src ?? '(unknown)';
@@ -553,6 +577,39 @@ function reportIssues(issuesList) {
553
577
  }
554
578
  }
555
579
 
580
+ /**
581
+ * Reports HTML validation issues to stderr, grouped by page URL.
582
+ * @param {HtmlValidateIssue[]} htmlIssues - Array of HTML validation issues to report
583
+ */
584
+ function reportHtmlValidation(htmlIssues) {
585
+ if (htmlIssues.length === 0) {
586
+ return;
587
+ }
588
+
589
+ console.error('\nHTML validation issues:\n');
590
+
591
+ // Group by page URL
592
+ /** @type {Map<string, HtmlValidateIssue[]>} */
593
+ const issuesByPage = new Map();
594
+ for (const issue of htmlIssues) {
595
+ const pageIssues = issuesByPage.get(issue.pageUrl) ?? [];
596
+ if (pageIssues.length === 0) {
597
+ issuesByPage.set(issue.pageUrl, pageIssues);
598
+ }
599
+ pageIssues.push(issue);
600
+ }
601
+
602
+ for (const [pageUrl, pageIssues] of issuesByPage.entries()) {
603
+ console.error(`Page ${chalk.cyan(pageUrl)}:`);
604
+ for (const issue of pageIssues) {
605
+ const severityLabel = issue.severity === 2 ? chalk.red('error') : chalk.yellow('warning');
606
+ console.error(
607
+ ` ${issue.location.line}:${issue.location.column} ${severityLabel} ${issue.message} ${chalk.gray(issue.ruleId)}`,
608
+ );
609
+ }
610
+ }
611
+ }
612
+
556
613
  /**
557
614
  * Crawls a website starting from seed URLs, discovering all internal links and checking for broken links/targets.
558
615
  * @param {CrawlOptions} rawOptions - Configuration options for the crawl
@@ -564,36 +621,91 @@ export async function crawl(rawOptions) {
564
621
 
565
622
  /** @type {AbortController | null} */
566
623
  let controller = null;
567
- if (options.startCommand) {
568
- console.log(chalk.blue(`Starting server with "${options.startCommand}"...`));
569
- controller = new AbortController();
570
- const appProcess = execaCommand(options.startCommand, {
571
- stdout: 'pipe',
572
- stderr: 'pipe',
573
- cancelSignal: controller.signal,
574
- env: {
575
- FORCE_COLOR: '1',
576
- ...process.env,
577
- },
578
- });
579
624
 
580
- // Prefix server logs
581
- const serverPrefix = chalk.gray('server: ');
582
- appProcess.stdout.pipe(prefixLines(serverPrefix)).pipe(process.stdout);
583
- appProcess.stderr.pipe(prefixLines(serverPrefix)).pipe(process.stderr);
584
- appProcess.catch(() => {});
625
+ try {
626
+ if (options.startCommand) {
627
+ console.log(chalk.blue(`Starting server with "${options.startCommand}"...`));
628
+ controller = new AbortController();
629
+ const appProcess = execaCommand(options.startCommand, {
630
+ stdout: 'pipe',
631
+ stderr: 'pipe',
632
+ cancelSignal: controller.signal,
633
+ env: {
634
+ FORCE_COLOR: '1',
635
+ ...process.env,
636
+ },
637
+ });
638
+
639
+ // Prefix server logs
640
+ const serverPrefix = chalk.gray('server: ');
641
+ appProcess.stdout.pipe(prefixLines(serverPrefix)).pipe(process.stdout);
642
+ appProcess.stderr.pipe(prefixLines(serverPrefix)).pipe(process.stderr);
643
+ appProcess.catch(() => {});
585
644
 
586
- await pollUrl(options.host, 10000);
645
+ // Poll the first page we are about to crawl (resolved against host) so we
646
+ // wait for the actual entry point to be serveable rather than the
647
+ // homepage, which may be a different (slower) page.
648
+ const healthcheckUrl = new URL(options.seedUrls[0] ?? '/', options.host).href;
649
+ await pollUrl(healthcheckUrl, SERVER_START_TIMEOUT);
587
650
 
588
- console.log(`Server started on ${chalk.underline(options.host)}`);
651
+ console.log(`Server started on ${chalk.underline(options.host)}`);
652
+ }
653
+
654
+ return await runCrawl(options, startTime);
655
+ } finally {
656
+ // Always stop the server, even when startup or crawling throws. Without
657
+ // this, a failed healthcheck (or any error) would leave the dev server
658
+ // running, which on slow environments (e.g. Netlify) leads to orphaned
659
+ // servers piling up across retries.
660
+ if (controller) {
661
+ console.log(chalk.blue('Stopping server...'));
662
+ controller.abort();
663
+ }
589
664
  }
665
+ }
590
666
 
667
+ /**
668
+ * Runs the crawl against an already-running server.
669
+ * @param {ResolvedCrawlOptions} options - Fully resolved crawl options
670
+ * @param {number} startTime - Timestamp (ms) when the crawl began, for duration reporting
671
+ * @returns {Promise<CrawlResult>} Crawl results including all links, pages, and issues found
672
+ */
673
+ async function runCrawl(options, startTime) {
591
674
  const knownTargets = await resolveKnownTargets(options);
592
675
 
593
676
  /** @type {Map<string, Promise<PageData>>} */
594
677
  const crawledPages = new Map();
595
678
  /** @type {Set<Link>} */
596
679
  const crawledLinks = new Set();
680
+ /** @type {Issue[]} */
681
+ const issues = [];
682
+ /**
683
+ * Spawns a crawl worker for a page URL.
684
+ * @param {string} pageUrl - The page URL to crawl
685
+ * @returns {Promise<{ pageData: PageData, links: Link[], htmlValidateResults: CrawlWorkerOutput['htmlValidateResults'] }>}
686
+ */
687
+ function crawlInWorker(pageUrl) {
688
+ return new Promise((resolve, reject) => {
689
+ /** @type {CrawlWorkerInput} */
690
+ const input = { pageUrl, options };
691
+ const worker = new Worker(crawlWorkerUrl, {
692
+ workerData: input,
693
+ });
694
+ worker.on('message', (/** @type {CrawlWorkerOutput} */ msg) => {
695
+ resolve({
696
+ pageData: {
697
+ url: msg.pageData.url,
698
+ status: msg.pageData.status,
699
+ targets: new Set(msg.pageData.targets),
700
+ contentType: msg.pageData.contentType,
701
+ },
702
+ links: msg.links,
703
+ htmlValidateResults: msg.htmlValidateResults,
704
+ });
705
+ });
706
+ worker.on('error', (err) => reject(err));
707
+ });
708
+ }
597
709
 
598
710
  const queue = new Queue(async (/** @type {Link} */ link) => {
599
711
  crawledLinks.add(link);
@@ -611,78 +723,30 @@ export async function crawl(rawOptions) {
611
723
  return;
612
724
  }
613
725
 
614
- const pagePromise = Promise.resolve().then(async () => {
615
- console.log(`Crawling ${chalk.cyan(pageUrl)}...`);
616
- const res = await fetch(new URL(pageUrl, options.host));
617
-
618
- const contentTypeHeader = res.headers.get('content-type');
619
- let type = 'text/html';
620
-
621
- if (contentTypeHeader) {
622
- try {
623
- const parsed = contentType.parse(contentTypeHeader);
624
- type = parsed.type;
625
- } catch {
626
- console.warn(
627
- chalk.yellow(`Warning: ${pageUrl} returned invalid content-type: ${contentTypeHeader}`),
628
- );
629
- }
630
- }
631
-
632
- /** @type {PageData} */
633
- const pageData = {
634
- url: pageUrl,
635
- status: res.status,
636
- targets: new Set(),
637
- contentType: type,
638
- };
639
-
640
- if (pageData.status < 200 || pageData.status >= 400) {
641
- console.warn(chalk.yellow(`Warning: ${pageUrl} returned status ${pageData.status}`));
642
- return pageData;
643
- }
644
-
645
- if (type.startsWith('image/')) {
646
- // Skip images
647
- return pageData;
648
- }
649
-
650
- if (type !== 'text/html' && type !== 'text/markdown') {
651
- console.warn(chalk.yellow(`Warning: ${pageUrl} returned non-HTML content-type: ${type}`));
652
- return pageData;
653
- }
654
-
655
- const rawContent = await res.text();
656
- const content = type === 'text/markdown' ? await markdownToHtml(rawContent) : rawContent;
657
-
658
- const dom = parse(content, { parseNoneClosedTags: true });
659
-
660
- let ignoredSelector = ':not(*)'; // matches nothing
661
- if (options.ignoredContent.length > 0) {
662
- ignoredSelector = Array.from(options.ignoredContent)
663
- .flatMap((selector) => [selector, `${selector} *`])
664
- .join(',');
665
- }
666
- const linksSelector = `a[href]:not(${ignoredSelector})`;
667
-
668
- const pageLinks = dom.querySelectorAll(linksSelector).map((a) => ({
669
- src: pageUrl,
670
- text: getAccessibleName(a, dom),
671
- href: a.getAttribute('href') ?? '',
672
- contentType: type,
673
- }));
674
-
675
- for (const target of dom.querySelectorAll('*[id]')) {
676
- if (!options.ignoredTargets.has(target.id)) {
677
- pageData.targets.add(`#${target.id}`);
726
+ console.log(`Crawling ${chalk.cyan(pageUrl)}...`);
727
+ const workerPromise = crawlInWorker(pageUrl);
728
+ const pagePromise = workerPromise.then((result) => {
729
+ if (result.htmlValidateResults) {
730
+ for (const validationResult of result.htmlValidateResults.results) {
731
+ for (const msg of validationResult.messages) {
732
+ issues.push({
733
+ type: 'html-validate',
734
+ message: msg.message,
735
+ pageUrl: result.htmlValidateResults.pageUrl,
736
+ ruleId: msg.ruleId,
737
+ severity: msg.severity,
738
+ location: { line: msg.line, column: msg.column },
739
+ selector: msg.selector,
740
+ });
741
+ }
678
742
  }
679
743
  }
680
744
 
681
- for (const pageLink of pageLinks) {
682
- queue.add(pageLink);
745
+ for (const discoveredLink of result.links) {
746
+ queue.add(discoveredLink);
683
747
  }
684
748
 
685
- return pageData;
749
+ return result.pageData;
686
750
  });
687
751
 
688
752
  crawledPages.set(pageUrl, pagePromise);
@@ -696,11 +760,6 @@ export async function crawl(rawOptions) {
696
760
 
697
761
  await queue.waitAll();
698
762
 
699
- if (controller) {
700
- console.log(chalk.blue('Stopping server...'));
701
- controller.abort();
702
- }
703
-
704
763
  const results = new Map(
705
764
  await Promise.all(
706
765
  Array.from(crawledPages.entries(), async ([a, b]) => /** @type {const} */ ([a, await b])),
@@ -711,10 +770,6 @@ export async function crawl(rawOptions) {
711
770
  await writePagesToFile(results, options.outPath);
712
771
  }
713
772
 
714
- /** Array to collect all issues found during validation */
715
- /** @type {Issue[]} */
716
- const issues = [];
717
-
718
773
  /** Count of links ignored due to ignores configuration */
719
774
  let ignoredCount = 0;
720
775
 
@@ -771,11 +826,24 @@ export async function crawl(rawOptions) {
771
826
  }
772
827
  }
773
828
 
774
- reportIssues(issues);
829
+ // Split issues by type for reporting
830
+ /** @type {BrokenLinkIssue[]} */
831
+ const brokenLinkIssues = /** @type {BrokenLinkIssue[]} */ (
832
+ issues.filter((issue) => issue.type === 'broken-link' || issue.type === 'broken-target')
833
+ );
834
+ /** @type {HtmlValidateIssue[]} */
835
+ const htmlValidateIssues = /** @type {HtmlValidateIssue[]} */ (
836
+ issues.filter((issue) => issue.type === 'html-validate')
837
+ );
838
+
839
+ reportBrokenLinks(brokenLinkIssues);
840
+ reportHtmlValidation(htmlValidateIssues);
775
841
 
776
842
  // Derive counts from issues
777
- const brokenLinks = issues.filter((issue) => issue.type === 'broken-link').length;
778
- const brokenLinkTargets = issues.filter((issue) => issue.type === 'broken-target').length;
843
+ const brokenLinks = brokenLinkIssues.filter((issue) => issue.type === 'broken-link').length;
844
+ const brokenLinkTargets = brokenLinkIssues.filter(
845
+ (issue) => issue.type === 'broken-target',
846
+ ).length;
779
847
 
780
848
  const endTime = Date.now();
781
849
  const durationSeconds = (endTime - startTime) / 1000;
@@ -784,14 +852,21 @@ export async function crawl(rawOptions) {
784
852
  unit: 'second',
785
853
  maximumFractionDigits: 2,
786
854
  }).format(durationSeconds);
855
+ const fmt = new Intl.NumberFormat('en-US').format;
787
856
  console.log(chalk.blue(`\nCrawl completed in ${duration}`));
788
- console.log(` Total links found: ${chalk.cyan(crawledLinks.size)}`);
789
- console.log(` Total broken links: ${chalk.cyan(brokenLinks)}`);
790
- console.log(` Total broken link targets: ${chalk.cyan(brokenLinkTargets)}`);
791
- console.log(` Total ignored: ${chalk.cyan(ignoredCount)}`);
857
+ console.log(` Total links found: ${chalk.cyan(fmt(crawledLinks.size))}`);
858
+ console.log(` Total broken links: ${chalk.cyan(fmt(brokenLinks))}`);
859
+ console.log(` Total broken link targets: ${chalk.cyan(fmt(brokenLinkTargets))}`);
860
+ console.log(` Total ignored: ${chalk.cyan(fmt(ignoredCount))}`);
861
+ if (options.htmlValidate.length > 0) {
862
+ const pagesWithHtmlIssues = new Set(htmlValidateIssues.map((issue) => issue.pageUrl)).size;
863
+ console.log(
864
+ ` HTML validation issues: ${chalk.cyan(fmt(htmlValidateIssues.length))} across ${chalk.cyan(fmt(pagesWithHtmlIssues))} ${pagesWithHtmlIssues === 1 ? 'page' : 'pages'}`,
865
+ );
866
+ }
792
867
 
793
868
  if (options.outPath) {
794
- console.log(chalk.blue(`Output written to: ${options.outPath}`));
869
+ console.log(chalk.blue(`Output written to: ${pathToFileURL(options.outPath)}`));
795
870
  }
796
871
 
797
872
  return { links: crawledLinks, pages: results, issues };
@@ -93,8 +93,12 @@ async function fetchCommitsRest({ octokit, repo, lastRelease, release, org = 'mu
93
93
  }
94
94
 
95
95
  const promises = results.map(async (commit) => {
96
- const prMatch = commit.commit.message.match(/#(\d+)/);
97
- if (prMatch === null) {
96
+ const matches = [...commit.commit.message.matchAll(/#(\d+)/g)];
97
+ // The PR number is always the last match.
98
+ // Sometimes the PR titles include an issue number like this:
99
+ // [tag] PR title (#00001) (#00002)
100
+ const prMatch = matches.at(-1);
101
+ if (!prMatch) {
98
102
  return null;
99
103
  }
100
104
 
@@ -125,6 +129,8 @@ async function fetchCommitsRest({ octokit, repo, lastRelease, release, org = 'mu
125
129
  association: getAuthorAssociation(pr.data.author_association),
126
130
  }
127
131
  : null,
132
+ prTitle: pr.data.title,
133
+ prBody: pr.data.body,
128
134
  });
129
135
  });
130
136
 
@@ -438,7 +438,7 @@ function renderContributors(contributors, config, lines) {
438
438
  const template = getTemplateString(
439
439
  config.contributors?.message?.contributors,
440
440
  allContributors.length,
441
- `${allContributors.length !== 1 ? 'All contributors of this release in alphabetical order' : 'Contributor of this release'} : {{contributors}}`,
441
+ `${allContributors.length !== 1 ? 'All contributors of this release in alphabetical order' : 'Contributor of this release'}: {{contributors}}`,
442
442
  );
443
443
  const contributorsMessage = templateString(template, {
444
444
  contributors: renderContributorsList(allContributors),
@@ -13,6 +13,8 @@ export interface FetchedCommitDetails {
13
13
  } | null;
14
14
  mergedAt: string | null;
15
15
  createdAt: string | null;
16
+ prTitle: string;
17
+ prBody: string;
16
18
  }
17
19
 
18
20
  /**
@@ -233,7 +235,7 @@ export interface IntroConfig {
233
235
  * - {{teamCount}}: Number of team members
234
236
  * - {{communityCount}}: Number of community contributors
235
237
  *
236
- * Example: "We'd like to extend a big thank you to the {{contributorCount}} contributors who made this release possible"
238
+ * Example: "A big thanks to the {{contributorCount}} contributors who made this release possible."
237
239
  *
238
240
  * Set to `false` or omit to disable the thank you message.
239
241
  */